业务本体与标准实体:跨源键值对齐、对账共变算子、健康证据链与实体助手

- 新增标准业务实体(CanonicalEntity)模型、确定性跨源编码翻译、后台候选实体扫描与文档驱动实体智能助手
- 新增指标本体图谱(factor / reconciles_with / upstream_driver)、DAG 防环校验以及 ontology_neighbors / source_health 内置工具
- 新增跨源对账(analysis_reconcile)与滞后共变(analysis_cochange)分析算子,打通 [[h:H1]] 数据源健康证据链、报告导出与本体消融评测
- 数据目录新增「标准业务实体」视图与候选审阅工作台,口径页集成指标本体拓扑面板与端到端浏览器检查
This commit is contained in:
ThearchyHelios
2026-11-12 12:57:53 +08:00
parent 2f90f0eb3a
commit 8d9a712eae
106 changed files with 16654 additions and 295 deletions
+10 -2
View File
@@ -58,7 +58,7 @@ def _op_mode(request: Mapping[str, Any]) -> tuple[str, str | None]:
def _compute(op: str, mode: str | None, raw: Mapping[str, Any], env: Any) -> AnalysisResult:
from app.analysis import comparable, decompose, differ, series
from app.analysis import cochange, comparable, decompose, differ, reconcile_op, series
if op == "decompose":
modes = {"additive": decompose.additive, "ratio": decompose.ratio, "factor": decompose.factor}
@@ -69,6 +69,10 @@ def _compute(op: str, mode: str | None, raw: Mapping[str, Any], env: Any) -> Ana
return series.trend(raw, env)
if op == "differ":
return differ.differ(raw, env)
if op == "reconcile":
return reconcile_op.reconcile(raw, env)
if op == "cochange":
return cochange.cochange(raw, env)
return comparable.comparable(raw, env)
@@ -83,7 +87,7 @@ def _env(op: str, mode: str | None, settings: Mapping[str, Any], inputs: Any, fa
def _data_refusal(op: str, mode: str | None, raw: Mapping[str, Any], env: Any, error: CallError) -> AnalysisResult:
"""随数据变化的调用错误记成的算不了的结论。params 记调用原样(label 照样消毒;调用还没走完、规范化不完整,复算拿它重走一遍
走到同一处),取过的快照都记进输入(Inputs.settle),阈值和判定标准同这个算子自己算不了时那一份。"""
from app.analysis import decompose, differ, series
from app.analysis import cochange, decompose, differ, reconcile_op, series
from app.analysis import phrases as ph
from app.analysis.common import build
@@ -97,6 +101,10 @@ def _data_refusal(op: str, mode: str | None, raw: Mapping[str, Any], env: Any, e
used, criterion = series.refusal_basis(op, env.settings)
elif op == "differ":
used, criterion = differ.refusal_basis(mode, env.settings)
elif op == "reconcile":
used, criterion = reconcile_op.refusal_basis(env.settings)
elif op == "cochange":
used, criterion = cochange.refusal_basis(env.settings)
else:
used, criterion = [], ph.CRITERION_COMPARABLE
code = "parent_refused" if error.code == "parent_refused" else "input_unavailable"
+414
View File
@@ -0,0 +1,414 @@
"""跨源伴随异动算子(cochange,W1-b):检验主指标异动时,上游关联指标(可跨数据源、可设领先滞后期 lag)是否在对应期次同步超出各自历史常规波动范围。
设计铁律(计划书决策 D6):
- 只检验两条序列各自相对自身稳健历史基线(中位数 + 1.4826·MAD)是否同步超出常规波动带(`k_warn` / `k_alert`),
不需要在小样本下硬算不可靠的相关系数 p 值;
- 系统渲染的判定语严格表述为「同期/领先期统计伴随异动(属关联共变线索,不直接构成因果证明)」,绝不输出「导致/致使」等硬因果断言;
- 全程有理数(Fraction)确定性计算,支持 `recompute` 逐字节复算(`contract.same`)。
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from fractions import Fraction
from typing import Any
from app.analysis import periods as pd
from app.analysis import phrases as ph
from app.analysis import stats
from app.analysis.common import Env, build, column_label, column_types, label_for, need, refuse, reserved_check
from app.analysis.contract import CONV_PCT, CallError
from app.analysis.inputs import Snap, same_value
from app.analysis.numbers import canon
from app.analysis.series import MAD_SCALE, unreadable_text
USED = ("k_warn", "k_alert", "min_history", "max_history")
FIXED = ("driver_period_val", "primary_value", "primary_eval", "driver_value", "driver_eval", "role")
_LAG_MAX = 12
def refusal_basis(settings: Mapping[str, Any]) -> tuple[list[str], str]:
k_w = ph.amount(settings["k_warn"])
min_h = settings["min_history"]
return list(USED), f"两侧序列各取至少 {min_h} 期历史,按中位数 ± {k_w} 倍稳健标准差(1.4826·MAD)检验是否同步超出常规波动"
@dataclass
class _ParsedSeries:
snap: Snap
period_col: str
measure_col: str
grain: str
label: str
by_ordinal: dict[int, tuple[pd.Period, Fraction]]
@dataclass
class _EvalOutcome:
target_period: pd.Period
target_val: Fraction
evaluated: Fraction # change rate or level
median: Fraction
sigma: Fraction
z: Fraction | None # None when sigma == 0
anomalous: bool
severe: bool
history_n: int
history_pairs: list[tuple[pd.Period, Fraction, Fraction]] # (period, raw_value, evaluated)
def _extract_series(
env: Env,
snap: Snap,
period_col: str,
measure_col: str,
*,
grain_hint: str | None,
label: str,
role_title: str,
) -> tuple[_ParsedSeries | None, tuple[str, str] | None]:
i_period = snap.index(period_col)
raw_periods = [row[i_period] if i_period < len(row) else None for row in snap.rows]
weeks = pd.week_scheme(snap.sql, period_col, snap.dialect)
parsed = pd.parse(raw_periods, grain_hint, weeks=weeks)
if parsed is None:
return None, ("period_unreadable", unreadable_text(period_col, raw_periods, grain_hint, weeks))
items, detected_grain = parsed
seen: set[int] = set()
by_ord: dict[int, tuple[pd.Period, Fraction]] = {}
for r, item in enumerate(items):
if item.ordinal in seen:
raise CallError(
"duplicate_rows",
ph.say(f"{snap.ref} 里同一期有不止一行:请先在 SQL 中按 ", column_label(env, snap, period_col), " 汇总到每期一行"),
)
seen.add(item.ordinal)
val = snap.number(r, measure_col, role_title)
if val is None:
continue
if pd.short_days(item, detected_grain) is not None:
continue
by_ord[item.ordinal] = (item, val)
return _ParsedSeries(snap, period_col, measure_col, detected_grain, label, by_ord), None
def _evaluate_at(
series: _ParsedSeries,
target_ord: int,
*,
basis: str,
min_history: int,
max_history: int,
k_warn: Fraction,
k_alert: Fraction,
who: str,
) -> tuple[_EvalOutcome | None, tuple[str, str] | None]:
if target_ord not in series.by_ordinal:
return None, ("target_gap", f"{who}缺少目标期次的数据,无法核对伴随波动")
step = 1
eval_points: list[tuple[pd.Period, Fraction, Fraction]] = []
for ord_k in sorted(series.by_ordinal):
if ord_k > target_ord:
continue
p_item, val = series.by_ordinal[ord_k]
if basis == "level":
eval_points.append((p_item, val, val))
else:
prev = series.by_ordinal.get(ord_k - step)
if prev is None:
if ord_k == target_ord:
return None, ("target_gap", f"{who}在目标期次「{p_item.raw}」的前一期缺失,无法计算环比变化率")
continue
if prev[1] == 0:
if ord_k == target_ord:
return None, ("zero_base", f"{who}在比较期「{prev[0].raw}」为 0,无法计算变化率(可改用 basis='level')")
continue
eval_points.append((p_item, val, (val - prev[1]) / abs(prev[1])))
if not eval_points or eval_points[-1][0].ordinal != target_ord:
return None, ("target_gap", f"{who}在目标期次无可用评估值")
target_tuple = eval_points[-1]
history = eval_points[:-1][-max_history:]
if len(history) < min_history:
return None, (
"history_short",
f"{who}可用历史只有 {len(history)} 期(至少需要 {min_history} 期)",
)
hist_vals = [x[2] for x in history]
med = stats.median(hist_vals)
mad = stats.mad(hist_vals, med)
sigma = MAD_SCALE * mad
curr = target_tuple[2]
diff = curr - med
if sigma == 0:
z = None
anomalous = diff != 0
severe = diff != 0
else:
z = diff / sigma
anomalous = abs(z) >= k_warn
severe = abs(z) >= k_alert
return _EvalOutcome(
target_period=target_tuple[0],
target_val=target_tuple[1],
evaluated=curr,
median=med,
sigma=sigma,
z=z,
anomalous=anomalous,
severe=severe,
history_n=len(history),
history_pairs=[*history, target_tuple],
), None
def cochange(raw: Mapping[str, Any], env: Env) -> Any:
k_warn = env.threshold("k_warn")
k_alert = env.threshold("k_alert")
min_history = int(env.settings["min_history"])
max_history = int(env.settings["max_history"])
_, criterion = refusal_basis(env.settings)
has_single = raw.get("query") not in (None, "")
has_two = raw.get("base_query") not in (None, "") or raw.get("target_query") not in (None, "")
if has_single and has_two:
raise CallError("conflicting_queries", "伴随异动检验不能同时写 query 和 base_query / target_query")
if not has_single and not has_two:
raise CallError("missing_query", "伴随异动检验请提供两份序列查询(base_query 与 target_query),或包含两列的单份查询(query)")
basis = str(raw.get("basis") or "change").strip()
if basis not in ("change", "level"):
raise CallError("bad_basis", "比较方式(basis)只能是 change(变化率)或 level(水平值)")
grain_hint = raw.get("grain")
if grain_hint is not None and grain_hint not in (*pd.GRAINS, "seq"):
raise CallError("bad_grain", "期次粒度(grain)只能写 day、week、month、quarter 或留空")
lag_raw = raw.get("lag")
if lag_raw in (None, ""):
lag = 0
elif isinstance(lag_raw, bool) or not isinstance(lag_raw, int) or not (0 <= lag_raw <= _LAG_MAX):
raise CallError("bad_lag", f"上游领先期数(lag)须为 0 到 {_LAG_MAX} 之间的整数(0 表示同期)")
else:
lag = lag_raw
params: dict[str, Any] = {"basis": basis}
if lag > 0:
params["lag"] = lag
if has_single:
snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号", role="data")
p_col = snap.column(need(raw, "period", "期次列"), "期次列")
reserved_check(p_col, FIXED, "期次列")
if raw.get("target_period") not in (None, ""):
raise CallError("unexpected_target_period", "单份查询(query)共用期次列 period,不需要写 target_period")
m_col = snap.column(need(raw, "measure", "主指标度量列"), "主指标度量列")
d_col = snap.column(need(raw, "driver_measure", "驱动度量列"), "驱动度量列")
if m_col == d_col:
raise CallError("same_column", f"主指标度量列与驱动度量列不能是同一列「{m_col}」")
params["query"] = snap.ref
params["period"] = p_col
params["measure"] = m_col
params["driver_measure"] = d_col
p_label = label_for(env, raw.get("label"), snap, m_col)
d_label = column_label(env, snap, d_col)
if raw.get("label") is not None:
params["label"] = ph.clean_label(raw.get("label")) or None
env.inputs.use(snap, "data", [p_col, m_col, d_col])
if snap.truncated:
return refuse(env, params, USED, "truncated", f"{snap.ref} 结果已被截断,无法核对完整时间序列", criterion=criterion)
p_series, err = _extract_series(env, snap, p_col, m_col, grain_hint=grain_hint, label=p_label, role_title="主指标度量列")
if err is not None or p_series is None:
return refuse(env, params, USED, err[0], err[1], criterion=criterion) # type: ignore[index]
d_series, err = _extract_series(env, snap, p_col, d_col, grain_hint=p_series.grain, label=d_label, role_title="驱动度量列")
if err is not None or d_series is None:
return refuse(env, params, USED, err[0], err[1], criterion=criterion) # type: ignore[index]
period_dim_type = snap.kind(p_col)
else:
p_snap = env.inputs.snap(need(raw, "base_query", "主指标查询编号"), "主指标查询编号", role="base")
d_snap = env.inputs.snap(need(raw, "target_query", "驱动指标查询编号"), "驱动指标查询编号", role="target")
p_col = p_snap.column(need(raw, "period", "主指标期次列"), "主指标期次列")
reserved_check(p_col, FIXED, "主指标期次列")
d_p_col = d_snap.column(raw.get("target_period") or p_col, "驱动期次列")
m_col = p_snap.column(need(raw, "measure", "主指标度量列"), "主指标度量列")
d_col = d_snap.column(need(raw, "driver_measure", "驱动度量列"), "驱动度量列")
params["base_query"] = p_snap.ref
params["target_query"] = d_snap.ref
params["period"] = p_col
if raw.get("target_period") not in (None, ""):
params["target_period"] = d_p_col
params["measure"] = m_col
params["driver_measure"] = d_col
p_label = label_for(env, raw.get("label"), p_snap, m_col)
d_label = column_label(env, d_snap, d_col)
if raw.get("label") is not None:
params["label"] = ph.clean_label(raw.get("label")) or None
env.inputs.use(p_snap, "base", [p_col, m_col])
env.inputs.use(d_snap, "target", [d_p_col, d_col])
if p_snap.truncated:
return refuse(env, params, USED, "truncated", f"{p_snap.ref} 结果已被截断,无法核对完整时间序列", criterion=criterion)
if d_snap.truncated:
return refuse(env, params, USED, "truncated", f"{d_snap.ref} 结果已被截断,无法核对完整时间序列", criterion=criterion)
p_series, err = _extract_series(env, p_snap, p_col, m_col, grain_hint=grain_hint, label=p_label, role_title="主指标度量列")
if err is not None or p_series is None:
return refuse(env, params, USED, err[0], err[1], criterion=criterion) # type: ignore[index]
d_series, err = _extract_series(env, d_snap, d_p_col, d_col, grain_hint=p_series.grain, label=d_label, role_title="驱动度量列")
if err is not None or d_series is None:
return refuse(env, params, USED, err[0], err[1], criterion=criterion) # type: ignore[index]
if d_series.grain != p_series.grain:
return refuse(
env,
params,
USED,
"incomparable_sides",
f"主指标期次粒度({p_series.grain})与驱动指标期次粒度({d_series.grain})不一致,无法对齐检验",
criterion=criterion,
)
period_dim_type = p_snap.kind(p_col)
params["grain"] = p_series.grain
if not p_series.by_ordinal:
return refuse(env, params, USED, "history_short", f"主指标「{p_label}」没有有效数值行", criterion=criterion)
wanted_target = raw.get("target")
if wanted_target in (None, ""):
target_ord = max(p_series.by_ordinal)
else:
matched_ords = [ord_k for ord_k, (p_it, _) in p_series.by_ordinal.items() if same_value(p_it.raw, wanted_target)]
if not matched_ords:
raise CallError(
"period_not_found",
f"主指标序列里没有期次「{wanted_target}」",
reason=f"主指标这次的数据里没有期次「{wanted_target}」",
)
target_ord = matched_ords[0]
params["target"] = str(p_series.by_ordinal[target_ord][0].raw)
p_eval, p_err = _evaluate_at(
p_series,
target_ord,
basis=basis,
min_history=min_history,
max_history=max_history,
k_warn=k_warn,
k_alert=k_alert,
who=f"主指标「{p_label}」",
)
if p_err is not None or p_eval is None:
return refuse(env, params, USED, p_err[0], p_err[1], criterion=criterion) # type: ignore[index]
driver_ord = target_ord - lag
d_eval, d_err = _evaluate_at(
d_series,
driver_ord,
basis=basis,
min_history=min_history,
max_history=max_history,
k_warn=k_warn,
k_alert=k_alert,
who=f"驱动指标「{d_label}」",
)
if d_err is not None or d_eval is None:
return refuse(env, params, USED, d_err[0], d_err[1], criterion=criterion) # type: ignore[index]
p_delta = p_eval.evaluated - p_eval.median
d_delta = d_eval.evaluated - d_eval.median
p_shown = ph.pct(p_eval.evaluated, signed=True) if basis == "change" else ph.amount(p_eval.evaluated)
d_shown = ph.pct(d_eval.evaluated, signed=True) if basis == "change" else ph.amount(d_eval.evaluated)
lag_word = f"领先 {lag} 期({d_eval.target_period.raw})" if lag > 0 else f"同期({d_eval.target_period.raw})"
if not p_eval.anomalous:
verdict_code = "primary_normal"
direction = None
phrase = ph.say(
f"主指标「{p_label}」在 {p_eval.target_period.raw} 处于自身常规波动范围内({p_shown}),无需触发跨源伴随异动归因。"
)
elif d_eval.anomalous:
verdict_code = "co_anomalous"
direction = "up" if p_delta > 0 else ("down" if p_delta < 0 else None)
same_sign = (p_delta > 0 and d_delta > 0) or (p_delta < 0 and d_delta < 0)
rel_word = "同向伴随异动" if same_sign else "反向伴随异动"
pz_txt = f",稳健 z = {ph.amount(p_eval.z, signed=True)}" if p_eval.z is not None else ""
dz_txt = f",稳健 z = {ph.amount(d_eval.z, signed=True)}" if d_eval.z is not None else ""
phrase = ph.say(
f"主指标「{p_label}」在 {p_eval.target_period.raw} 超出常规波动范围({p_shown}{pz_txt}),"
f"上游关联指标「{d_label}」在{lag_word}亦超出自身常规波动范围({d_shown}{dz_txt}),"
f"两者呈{rel_word}(属统计共变线索,不直接构成因果证明)。"
)
else:
verdict_code = "target_normal"
direction = "up" if p_delta > 0 else ("down" if p_delta < 0 else None)
phrase = ph.say(
f"主指标「{p_label}」在 {p_eval.target_period.raw} 超出常规波动范围({p_shown}),"
f"但上游关联指标「{d_label}」在{lag_word}处于常规波动范围内({d_shown}),未见伴随异动。"
)
# 组装对齐结果表(最近最多 12 期历史 + 目标期)
p_hist_map = {pt[0].ordinal: pt for pt in p_eval.history_pairs}
d_hist_map = {pt[0].ordinal: pt for pt in d_eval.history_pairs}
recent_ords = [pt[0].ordinal for pt in p_eval.history_pairs[-12:]]
rows_out: list[list[Any]] = []
for ord_k in recent_ords:
p_pt = p_hist_map[ord_k]
d_pt = d_hist_map.get(ord_k - lag)
role_txt = "当期" if ord_k == target_ord else "历史基线"
rows_out.append([
p_pt[0].raw,
d_pt[0].raw if d_pt else None,
canon(p_pt[1]),
canon(p_pt[2]),
canon(d_pt[1]) if d_pt else None,
canon(d_pt[2]) if d_pt else None,
role_txt,
])
cols = [p_col, *FIXED]
labels_map = {
p_col: "主指标期次",
"driver_period_val": "驱动指标期次",
"primary_value": p_label,
"primary_eval": "主指标变化率" if basis == "change" else "主指标水平",
"driver_value": d_label,
"driver_eval": "驱动变化率" if basis == "change" else "驱动水平",
"role": "期次角色",
}
table_formats = {"primary_eval": CONV_PCT, "driver_eval": CONV_PCT} if basis == "change" else {}
named_formats = {"primary_evaluated": CONV_PCT, "driver_evaluated": CONV_PCT} if basis == "change" else {}
table = {
"columns": cols,
"labels": labels_map,
"rows": rows_out,
"column_types": column_types(cols, dims={p_col: period_dim_type}, texts=("driver_period_val", "role")),
}
values: dict[str, Any] = {
"primary_value": canon(p_eval.target_val),
"primary_evaluated": canon(p_eval.evaluated),
"primary_median": canon(p_eval.median),
"primary_z": canon(p_eval.z),
"driver_value": canon(d_eval.target_val),
"driver_evaluated": canon(d_eval.evaluated),
"driver_median": canon(d_eval.median),
"driver_z": canon(d_eval.z),
"history_periods": str(min(p_eval.history_n, d_eval.history_n)),
"lag_periods": str(lag),
}
verdict = {"code": verdict_code, "subjects": [[d_label]] if verdict_code == "co_anomalous" else [], "direction": direction}
return build(
env,
params,
USED,
table=table,
values=values,
verdict=verdict,
criterion=criterion,
phrase=phrase,
formats=table_formats,
named=named_formats,
)
+7 -2
View File
@@ -32,8 +32,8 @@ NODE_TYPE = "analysis"
#: 设置分组的键:settings 里的 analysis 一组、run.options["analysis"]
SETTINGS_KEY = "analysis"
Op = Literal["decompose", "baseline", "trend", "differ", "comparable"]
OPS: tuple[str, ...] = ("decompose", "baseline", "trend", "differ", "comparable")
Op = Literal["decompose", "baseline", "trend", "differ", "comparable", "reconcile", "cochange"]
OPS: tuple[str, ...] = ("decompose", "baseline", "trend", "differ", "comparable", "reconcile", "cochange")
#: 有模式的算子和它们的模式;其余算子 mode 为 None
MODES: dict[str, tuple[str, ...]] = {
"decompose": ("additive", "ratio", "factor"),
@@ -50,6 +50,8 @@ METHODS: dict[tuple[str, str | None], str] = {
("differ", "mean"): "differ.welch@2",
("differ", "rows"): "differ.welch_rows@2",
("comparable", None): "comparable@2",
("reconcile", None): "reconcile@1",
("cochange", None): "cochange.mad@1",
}
#: agent 分析工具的名字(tools/builtin 里注册),一个算子一个工具:参数各自聚焦,模型少写错
TOOL_NAMES: dict[str, str] = {op: f"analysis_{op}" for op in OPS}
@@ -57,6 +59,7 @@ TOOL_NAMES: dict[str, str] = {op: f"analysis_{op}" for op in OPS}
#: (engine/labels 的 ANALYSIS_OP_LABEL / ANALYSIS_MODE_LABEL 引用这里)、「{叫法}没有做成」的判定语都用它(method_label 拼成一句)
OP_LABELS: dict[str, str] = {
"decompose": "变化拆解", "baseline": "波动基线", "trend": "趋势判断", "differ": "差异检验", "comparable": "可比性检查",
"reconcile": "跨源对账", "cochange": "伴随异动",
}
MODE_LABELS: dict[str, str] = {
"additive": "可加", "ratio": "比率", "factor": "乘积", "rate": "比率", "mean": "均值", "rows": "逐行明细",
@@ -82,6 +85,8 @@ VERDICTS: dict[str, tuple[str, ...]] = {
"trend": ("rising", "falling", "no_trend"),
"differ": ("differs", "no_evidence"),
"comparable": ("comparable", "conditional", "incomparable"),
"reconcile": ("aligned", "diverged", "missing_counterpart"),
"cochange": ("co_anomalous", "target_normal", "primary_normal"),
}
#: 算不了的原因(细案 01 §8.2)。算不了也记成结论:写作者可以照实写「历史只有 5 周,判断不了是否异常」。最后两种是分析卡
#: 里随数据变化的调用错误记成的(analysis.run 的 refuse_data_errors,见 app/analysis 的 DATA_ERRORS):parent_refused 上一层
+336
View File
@@ -0,0 +1,336 @@
"""跨源对账算子(reconcile,W1-b):对齐同一份或两份查询快照上的镜像指标,按容差判定对账一致、存在偏差或对侧缺失。
输入方式:
- 两份跨源快照:`base_query`(主源/左侧)+ `target_query`(对账源/右侧),`measure`(左侧度量列),可选 `target_measure`
(右侧度量列,缺省同 `measure`)、`by`(左侧对齐键列,0..2 列)、`target_by`(右侧对齐键列,缺省同 `by`)、
`unit_factor`(物理计量单位换算系数,右侧乘以该系数后与左侧比较,缺省 1);
- 单份已合并快照:`query` + `measure`(左侧列)+ `target_measure`(右侧列),可选 `by`、`unit_factor`。
阈值契约(D4):请求参数不收容差阈值,统一使用平台分析设置里的 `flat_pct`(相对容差,缺省 0.5%)与 `table_top_k`。
纯函数,全程精确有理数(Fraction),支持 `recompute` 逐字节复算核对(`contract.same`)。
"""
from __future__ import annotations
import math
from collections.abc import Mapping
from fractions import Fraction
from typing import Any
from app.analysis import phrases as ph
from app.analysis.common import Env, build, column_label, column_types, label_for, need, refuse, reserved_check
from app.analysis.contract import CONV_PCT, CallError
from app.analysis.inputs import Snap, as_list, hashable
from app.analysis.numbers import canon, to_fraction
USED = ("flat_pct", "table_top_k")
FIXED = ("left_value", "right_value", "diff", "rel_diff", "row_status")
_MAX_BY = 2
def refusal_basis(settings: Mapping[str, Any]) -> tuple[list[str], str]:
tol = ph.pct(settings["flat_pct"], signed=False)
return list(USED), f"按相对误差不超过 {tol} 核对两侧口径数值是否对齐"
def _parse_unit_factor(raw_val: Any) -> tuple[Fraction, float | int | None]:
if raw_val in (None, ""):
return Fraction(1), None
if isinstance(raw_val, bool) or not isinstance(raw_val, (int, float)):
f = to_fraction(raw_val)
if f is None or f <= 0:
raise CallError("bad_unit_factor", "单位换算系数(unit_factor)须为大于 0 的有限数字(如吨转千克写 1000)")
as_num: float | int = int(f) if f.denominator == 1 else float(f)
return f, as_num
if not math.isfinite(float(raw_val)) or float(raw_val) <= 0:
raise CallError("bad_unit_factor", "单位换算系数(unit_factor)须为大于 0 的有限数字(如吨转千克写 1000)")
f = to_fraction(raw_val)
assert f is not None and f > 0
as_num = int(raw_val) if isinstance(raw_val, int) else float(raw_val)
return f, as_num
def _norm_key(raw_key: tuple[Any, ...]) -> tuple[str, ...]:
return tuple("" if v is None else str(hashable(v)).strip() for v in raw_key)
def _extract_side(
env: Env,
snap: Snap,
by_cols: list[str],
measure_col: str,
*,
role_name: str,
) -> dict[tuple[str, ...], tuple[list[Any], Fraction | None]]:
out: dict[tuple[str, ...], tuple[list[Any], Fraction | None]] = {}
if not by_cols:
if len(snap.rows) > 1:
raise CallError(
"duplicate_rows",
f"{snap.ref} 有 {len(snap.rows)} 行,未指定对齐维度列(by)时每侧只能是汇总后的 1 行;请加上 by 列或在 SQL 中汇总",
)
val = snap.number(0, measure_col, role_name) if snap.rows else None
out[()] = ([], val)
return out
idx_by = [snap.index(c) for c in by_cols]
for r in range(len(snap.rows)):
row = snap.rows[r]
raw_vals = [row[i] if i < len(row) else None for i in idx_by]
key = _norm_key(tuple(raw_vals))
if key in out:
shown = ph.key_text(raw_vals)
cols_label = "、".join(column_label(env, snap, c) for c in by_cols)
raise CallError(
"duplicate_rows",
f"{snap.ref} 在维度 {cols_label} = {shown} 上有多行:请先在 SQL 中按对齐维度汇总到唯一一行",
)
val = snap.number(r, measure_col, role_name)
out[key] = (raw_vals, val)
return out
def reconcile(raw: Mapping[str, Any], env: Env) -> Any:
tol = env.threshold("flat_pct")
top_k = int(env.settings["table_top_k"])
_, criterion = refusal_basis(env.settings)
has_single = raw.get("query") not in (None, "")
has_two = raw.get("base_query") not in (None, "") or raw.get("target_query") not in (None, "")
if has_single and has_two:
raise CallError("conflicting_queries", "跨源对账不能同时写 query 和 base_query / target_query:单表对账写 query,两份快照对账写 base_query 与 target_query")
if not has_single and not has_two:
raise CallError("missing_query", "跨源对账请提供两份查询编号(base_query 与 target_query),或单份合并查询编号(query)")
by_raw = [str(c).strip() for c in as_list(raw.get("by")) if str(c).strip()]
if len(by_raw) > _MAX_BY:
raise CallError("too_many_by", f"对账对齐维度最多支持 {_MAX_BY} 列")
factor_frac, factor_param = _parse_unit_factor(raw.get("unit_factor"))
params: dict[str, Any] = {}
if has_single:
snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号", role="data")
by_cols = [snap.column(c, "对齐维度列") for c in by_raw]
for c in by_cols:
reserved_check(c, FIXED, "对齐维度列")
if raw.get("target_by") not in (None, [], ""):
raise CallError("unexpected_target_by", "单份查询(query)对账共用同一组维度列 by,不需要写 target_by")
left_col = snap.column(need(raw, "measure", "左侧度量列"), "左侧度量列")
right_col = snap.column(need(raw, "target_measure", "对侧度量列"), "对侧度量列")
if left_col == right_col and factor_frac == 1:
raise CallError("same_column", f"单表对账的左侧度量列与对侧度量列不能是同一列「{left_col}」")
params["query"] = snap.ref
if by_cols:
params["by"] = by_cols
params["measure"] = left_col
params["target_measure"] = right_col
if factor_param is not None:
params["unit_factor"] = factor_param
label = label_for(env, raw.get("label"), snap, left_col)
if raw.get("label") is not None:
params["label"] = ph.clean_label(raw.get("label")) or None
env.inputs.use(snap, "data", [*by_cols, left_col, right_col])
if snap.truncated:
return refuse(env, params, USED, "truncated", f"{snap.ref} 的查询结果已被截断,无法做完整对账", criterion=criterion)
left_map = _extract_side(env, snap, by_cols, left_col, role_name="左侧度量列")
right_map = _extract_side(env, snap, by_cols, right_col, role_name="对侧度量列")
dim_types = {c: snap.kind(c) for c in by_cols}
dim_labels = {c: column_label(env, snap, c) for c in by_cols}
else:
left_snap = env.inputs.snap(need(raw, "base_query", "主侧查询编号"), "主侧查询编号", role="base")
right_snap = env.inputs.snap(need(raw, "target_query", "对侧查询编号"), "对侧查询编号", role="target")
left_by = [left_snap.column(c, "对齐维度列") for c in by_raw]
for c in left_by:
reserved_check(c, FIXED, "对齐维度列")
t_by_raw = [str(c).strip() for c in as_list(raw.get("target_by")) if str(c).strip()]
if t_by_raw and len(t_by_raw) != len(left_by):
raise CallError("target_by_mismatch", f"对侧维度列(target_by,{len(t_by_raw)} 列)数量须与对齐维度列(by,{len(left_by)} 列)一致")
right_by = [right_snap.column(c, "对侧维度列") for c in (t_by_raw or by_raw)]
left_col = left_snap.column(need(raw, "measure", "左侧度量列"), "左侧度量列")
right_col = right_snap.column(raw.get("target_measure") or left_col, "对侧度量列")
params["base_query"] = left_snap.ref
params["target_query"] = right_snap.ref
if left_by:
params["by"] = left_by
if t_by_raw:
params["target_by"] = right_by
params["measure"] = left_col
if raw.get("target_measure") not in (None, ""):
params["target_measure"] = right_col
if factor_param is not None:
params["unit_factor"] = factor_param
label = label_for(env, raw.get("label"), left_snap, left_col)
if raw.get("label") is not None:
params["label"] = ph.clean_label(raw.get("label")) or None
env.inputs.use(left_snap, "base", [*left_by, left_col])
env.inputs.use(right_snap, "target", [*right_by, right_col])
if left_snap.truncated:
return refuse(env, params, USED, "truncated", f"{left_snap.ref} 的查询结果已被截断,无法做完整对账", criterion=criterion)
if right_snap.truncated:
return refuse(env, params, USED, "truncated", f"{right_snap.ref} 的查询结果已被截断,无法做完整对账", criterion=criterion)
left_map = _extract_side(env, left_snap, left_by, left_col, role_name="左侧度量列")
right_map = _extract_side(env, right_snap, right_by, right_col, role_name="对侧度量列")
by_cols = left_by
dim_types = {c: left_snap.kind(c) for c in left_by}
dim_labels = {c: column_label(env, left_snap, c) for c in left_by}
all_keys = sorted(set(left_map) | set(right_map))
paired_records: list[dict[str, Any]] = []
left_sum = Fraction(0)
right_sum = Fraction(0)
matched_count = 0
diverged_count = 0
missing_count = 0
max_abs = Fraction(0)
max_rel = Fraction(0)
for key in all_keys:
l_entry = left_map.get(key)
r_entry = right_map.get(key)
raw_dims = (l_entry[0] if l_entry is not None else (r_entry[0] if r_entry is not None else []))
l_val = l_entry[1] if l_entry is not None else None
r_raw = r_entry[1] if r_entry is not None else None
r_val = (r_raw * factor_frac) if r_raw is not None else None
if l_val is None and r_val is None:
continue
if l_val is not None:
left_sum += l_val
if r_val is not None:
right_sum += r_val
if l_val is None or r_val is None:
missing_count += 1
status_code = "missing_right" if l_val is not None else "missing_left"
sort_gap = abs(l_val if l_val is not None else r_val) # type: ignore[arg-type]
paired_records.append({
"dims": raw_dims,
"key": key,
"left": l_val,
"right": r_val,
"diff": None,
"rel": None,
"status": status_code,
"sort_rank": (2, sort_gap),
})
continue
matched_count += 1
diff = l_val - r_val
abs_diff = abs(diff)
denom = max(abs(l_val), abs(r_val))
rel = (abs_diff / denom) if denom != 0 else Fraction(0)
is_div = (abs_diff > tol * denom) if denom != 0 else False
if is_div:
diverged_count += 1
if abs_diff > max_abs:
max_abs = abs_diff
if rel > max_rel:
max_rel = rel
paired_records.append({
"dims": raw_dims,
"key": key,
"left": l_val,
"right": r_val,
"diff": diff,
"rel": rel,
"status": "diverged" if is_div else "aligned",
"sort_rank": (1 if is_div else 0, abs_diff),
})
if not paired_records:
return refuse(env, params, USED, "too_few_segments", "两侧查询均无有效数值行,无法对账", criterion=criterion)
paired_records.sort(key=lambda item: (-item["sort_rank"][0], -item["sort_rank"][1], item["key"]))
net_diff = left_sum - right_sum
net_denom = max(abs(left_sum), abs(right_sum))
net_rel = (abs(net_diff) / net_denom) if net_denom != 0 else Fraction(0)
if diverged_count == 0 and missing_count > 0:
verdict_code = "missing_counterpart"
direction = None
missing_recs = [r for r in paired_records if r["status"] != "aligned"]
subjects = [[str(x) for x in r["dims"]] for r in missing_recs[:3] if r["dims"]]
matched_note = f"(已配对的 {matched_count} 项数值一致)" if matched_count > 0 else ""
phrase = ph.say(
f"「{label}」跨源对账发现对侧缺失(共 {missing_count} 项未能配对){matched_note},两侧无法完全闭合,请优先排查对侧数据源同步缺口或主数据编码未映射。"
)
elif diverged_count == 0 and missing_count == 0:
verdict_code = "aligned"
direction = None
subjects = []
phrase = ph.say(
f"「{label}」两侧对账一致(合计 {ph.amount(left_sum)} 对 {ph.amount(right_sum)},最大相对偏差 {ph.pct(max_rel, signed=False)},在容差 {ph.pct(tol, signed=False)} 以内)。"
)
else:
verdict_code = "diverged"
direction = "up" if net_diff > 0 else ("down" if net_diff < 0 else None)
flagged = [r for r in paired_records if r["status"] != "aligned"]
subjects = [[str(x) for x in r["dims"]] for r in flagged[:3] if r["dims"]]
extra_missing = f",另有 {missing_count} 项对侧缺失" if missing_count else ""
top_hint = ""
if subjects:
top_hint = f"(偏差集中在 {'、'.join(ph.key_text(s) for s in subjects)})"
phrase = ph.say(
f"「{label}」跨源对账存在显著偏差:主侧合计 {ph.amount(left_sum)},对侧合计 {ph.amount(right_sum)},净差额 {ph.amount(net_diff, signed=True)}"
f"(最大相对偏差 {ph.pct(max_rel, signed=False)},超出容差 {ph.pct(tol, signed=False)},共 {diverged_count} 项超差{extra_missing}){top_hint}。"
)
cols = [*by_cols, *FIXED]
labels_map = {
**dim_labels,
"left_value": "主侧数值",
"right_value": "对侧数值",
"diff": "差额",
"rel_diff": "相对偏差",
"row_status": "对账状态",
}
status_cn = {
"aligned": "一致",
"diverged": "超差",
"missing_right": "对侧缺失",
"missing_left": "主侧缺失",
}
shown_rows = [
[
*rec["dims"],
canon(rec["left"]),
canon(rec["right"]),
canon(rec["diff"]),
canon(rec["rel"]),
status_cn[rec["status"]],
]
for rec in paired_records[:top_k]
]
table = {
"columns": cols,
"labels": labels_map,
"rows": shown_rows,
"column_types": column_types(cols, dims=dim_types, texts=("row_status",)),
}
values = {
"left_total": canon(left_sum),
"right_total": canon(right_sum),
"net_diff": canon(net_diff),
"net_rel_diff": canon(net_rel),
"max_abs_diff": canon(max_abs),
"max_rel_diff": canon(max_rel),
"matched_rows": str(matched_count),
"diverged_rows": str(diverged_count),
"missing_counterpart_rows": str(missing_count),
}
verdict: dict[str, Any] = {"code": verdict_code, "subjects": subjects, "direction": direction}
return build(
env,
params,
USED,
table=table,
values=values,
verdict=verdict,
criterion=criterion,
phrase=phrase,
formats={"rel_diff": CONV_PCT},
named={"net_rel_diff": CONV_PCT, "max_rel_diff": CONV_PCT},
)
+52 -3
View File
@@ -19,6 +19,11 @@ _PHRASES = {
"no_evidence": "西南和华东的退款率差异可能只是样本波动(样本量 40、230)",
"comparable": "两期可比",
"incomparable": "两期不可比:本期只有 3 天数据",
"aligned": "两侧报废数对账一致(主侧 120,对账侧 120,相差 0)",
"diverged": "两侧报废数存在显著偏差(主侧 120,对账侧 95,相差 +25,+26.3%),偏差集中在 L2",
"missing_counterpart": "两侧报废数对账发现对侧缺失记录(共 1 项缺失:L4)",
"co_anomalous": "产线效率本期变化 -14.2% 超出常规波动,关联的非计划停机在同期同样超出其常规波动(+185.0%,同期同步异动,统计伴随关系,非因果断言)",
"target_normal": "产线效率本期变化 -14.2% 超出常规波动,但关联的非计划停机在同期处于其常规波动范围内(+2.1%)",
}
_WEEKS = ["2026-07-20", "2026-07-27", "2026-08-03", "2026-08-10", "2026-08-17", "2026-08-24", "2026-08-31",
@@ -102,13 +107,57 @@ def _comparable() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[
return table, values, params, ["gmv"]
def _reconcile() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]:
columns = ["line_code", "base", "target", "scaled_target", "diff", "rel_diff", "status"]
table = {
"columns": columns,
"labels": {"line_code": "产线", "base": "主侧", "target": "对账侧原值", "scaled_target": "换算后对账侧",
"diff": "差值", "rel_diff": "相对偏差", "status": "结果"},
"rows": [["L2", "60", "35", "35", "25", "0.714285714286", "超差"],
["L1", "60", "60", "60", "0", "0", "一致"]],
"column_types": {"line_code": "text", "base": "number", "target": "number", "scaled_target": "number",
"diff": "number", "rel_diff": "number", "status": "text"},
}
values = {"base_total": "120", "target_total": "95", "diff_total": "25", "rel_diff_total": "0.263157894737",
"unit_factor": "1", "matched_count": "2", "diverged_count": "1", "missing_base_count": "0",
"missing_target_count": "0", "max_abs_diff": "25"}
params = {"base_query": "Q1", "target_query": "Q2", "by": ["line_code"], "measure": "scrap_qty",
"target_measure": "scrap_units", "unit_factor": "1", "label": "报废数"}
return table, values, params, ["line_code", "scrap_qty"]
def _cochange() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]:
columns = ["day", "primary_value", "primary_change", "driver_value", "driver_change", "role"]
rows = [[w, "92", "0.01", "12", "0.02", "历史"] for w in _WEEKS[:-1]] + [
[_WEEKS[-1], "79", "-0.141304347826", "34", "1.833333333333", "本期"]
]
table = {
"columns": columns,
"labels": {"day": "日期", "primary_value": "产线效率", "primary_change": "主指标变化率",
"driver_value": "非计划停机", "driver_change": "关联指标变化率", "role": "说明"},
"rows": rows,
"column_types": {"day": "text", "primary_value": "number", "primary_change": "number",
"driver_value": "number", "driver_change": "number", "role": "text"},
}
values = {"primary_x": "-0.141304347826", "primary_z": "-4.2", "primary_band_lo": "-0.04",
"primary_band_hi": "0.04", "driver_x": "1.833333333333", "driver_z": "5.8",
"driver_band_lo": "-0.15", "driver_band_hi": "0.15", "lag": "0", "n_history": "12",
"co_direction": "opposite"}
params = {"base_query": "Q1", "target_query": "Q2", "period": "day", "measure": "efficiency",
"driver_measure": "unplanned_min", "lag": 0, "basis": "change", "target": _WEEKS[-1],
"label": "产线效率"}
return table, values, params, ["day", "efficiency"]
_SHAPES = {"decompose": _decompose, "baseline": _baseline, "trend": _trend, "differ": _differ,
"comparable": _comparable}
"comparable": _comparable, "reconcile": _reconcile, "cochange": _cochange}
_CRITERIA = {"decompose": "一段占总变化一半以上算主要来源",
"baseline": "本期和过去 12 周的中位数比,差不到 2 倍稳健标准差算常规波动",
"trend": "Mann–Kendall 检验的双侧 p 值小于 0.05 时按 Theil–Sen 斜率的正负判上升或下降",
"differ": "双侧 p 值小于 0.05 算差异站得住",
"comparable": "只差日期、窗口一样长、是整期、星期构成相同才算可比"}
"comparable": "只差日期、窗口一样长、是整期、星期构成相同才算可比",
"reconcile": "两侧总量及各对齐切片相对偏差均不超过 1.0% 且无缺失项算对账一致",
"cochange": "主指标与关联指标在对齐期次上同按各自历史中位数 ± 2.0 倍稳健标准差检验是否同步超出常规波动"}
def fake_result(op: str = "decompose", mode: str | None = "additive", verdict: str | None = "driver", *,
@@ -121,7 +170,7 @@ def fake_result(op: str = "decompose", mode: str | None = "additive", verdict: s
elif op == "differ" and mode not in ("rate", "mean", "rows"):
mode = "rate"
table, values, params, columns = _SHAPES.get(op, _decompose)()
if op == "comparable":
if op in ("comparable", "reconcile", "cochange"):
default_inputs = [{"ref": ref, "artifact": "0" * 64, "role": role, "columns": columns, "rows": "all"}
for ref, role in (("Q4", "base"), ("Q5", "target"))]
else: