mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:11:59 +00:00
五个分析算子与 run / recompute（细案 01 §1–§8）：decompose（additive 按维度算贡献、headline 与下钻闭合、抵消、新出现 / 消失；ratio 中点对称拆结构与自身效应、辛普森、分母小；factor LMDI-I；两份快照先做可比性，口径结构不同不拆）、baseline（中位数与 MAD 的稳健 z、σ̂ = 0、日数据 lag 7、新高新低）、trend（Theil–Sen + Mann–Kendall、末尾连续期数）、differ（rate 的 z 检验 / Fisher、mean 与 rows 的 Welch、Holm、rows 的敏感性）、comparable（同口径、窗口长度、整期、星期构成、数据问题，现查记进 probes）；run 先登记现查、查完再算，运行当天和目录事实记进 params，recompute 只用记下的、不现查；判定码、原因码、附带说明码都在契约里；输入记下方言和遮罩列名，复算逐字段相同；测试用虚构零售库，每个用例都核对复算
This commit is contained in:
ThearchyHelios
@@ -8,14 +8,24 @@
|
||||
- recompute：按结论里记下的 params、settings、probes 和输入快照（按 sha256 取回）重算一遍，不现查；和封存的用
|
||||
contract.same 比。
|
||||
|
||||
契约（名字、形状、缺省阈值）在 contract.py。
|
||||
契约（名字、形状、缺省阈值）在 contract.py。各算子在 decompose.py（变化拆解）、series.py（波动基线、趋势）、differ.py
|
||||
（差异检验）、comparable.py（可比性检查）；统计在 stats.py，判定语和数的写法在 phrases.py，现查在 probe.py。
|
||||
|
||||
**一份结论只由 inputs（快照里的格）、probes、params、settings、method 决定**：run 时从 context 读到的东西（运行当天、目录
|
||||
里的度量类型和叫法、现查的范围）都记进结论——运行当天、目录事实在 params（today、catalog），现查结果在 probes——复算只读
|
||||
记下的。同样的输入两次 run 逐字节相同（现查结果相同时）；recompute 和 run 逐字段相同。
|
||||
|
||||
**参数**（AnalysisRequest.params；列名不分大小写，按快照认）见各算子文件头；params 里的 catalog、today、depth 由系统填，
|
||||
调用方写了也不收。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
from app.analysis.contract import AnalysisRequest, AnalysisResult, Context, InputSnapshot
|
||||
from app.analysis.contract import (MODES, OPS, AnalysisRequest, AnalysisResult, CallError, Context, InputSnapshot,
|
||||
effective_settings)
|
||||
|
||||
#: 按调用里写的名字取回输入快照；没有这个名字返回 None
|
||||
Resolver = Callable[[str], InputSnapshot | None]
|
||||
@@ -23,16 +33,144 @@ Resolver = Callable[[str], InputSnapshot | None]
|
||||
ParentResolver = Callable[[str], AnalysisResult | None]
|
||||
|
||||
|
||||
def _op_mode(request: Mapping[str, Any]) -> tuple[str, str | None]:
|
||||
from app.analysis.phrases import OP_NAMES
|
||||
|
||||
op = request.get("op")
|
||||
if op not in OPS:
|
||||
raise CallError("unknown_op", f"不认识的分析：{op}。能做的有{'、'.join(OP_NAMES[o] for o in OPS)}")
|
||||
modes = MODES.get(op)
|
||||
mode = request.get("mode")
|
||||
if modes is None:
|
||||
return op, None
|
||||
if mode not in modes:
|
||||
raise CallError("unknown_mode", f"{OP_NAMES[op]}要写模式：{' / '.join(modes)}")
|
||||
return op, mode
|
||||
|
||||
|
||||
def _compute(op: str, mode: str | None, raw: Mapping[str, Any], env: Any) -> AnalysisResult:
|
||||
from app.analysis import comparable, decompose, differ, series
|
||||
|
||||
if op == "decompose":
|
||||
return {"additive": decompose.additive, "ratio": decompose.ratio, "factor": decompose.factor}[mode](raw, env) # type: ignore[index]
|
||||
if op == "baseline":
|
||||
return series.baseline(raw, env)
|
||||
if op == "trend":
|
||||
return series.trend(raw, env)
|
||||
if op == "differ":
|
||||
return differ.differ(raw, env)
|
||||
return comparable.comparable(raw, env)
|
||||
|
||||
|
||||
def _env(op: str, mode: str | None, settings: Mapping[str, Any], inputs: Any, facts: Any, book: Any,
|
||||
today: date | None, parents: ParentResolver | None, hypothesis: Any) -> Any:
|
||||
from app.analysis.common import Env
|
||||
|
||||
return Env(op=op, mode=mode, settings=settings, inputs=inputs, facts=facts, book=book, today=today,
|
||||
parents=parents, hypothesis=hypothesis if isinstance(hypothesis, str) else None)
|
||||
|
||||
|
||||
async def run(request: AnalysisRequest, resolve: Resolver, *, settings: Mapping[str, Any], context: Context,
|
||||
parents: ParentResolver | None = None) -> AnalysisResult:
|
||||
"""算一份分析结论。settings 是 contract.effective_settings 算好的阈值。"""
|
||||
raise NotImplementedError("analysis.run：第 1 波 W1-a 实现")
|
||||
from app.analysis.common import DERIVED_PARAMS
|
||||
from app.analysis.facts import Facts
|
||||
from app.analysis.inputs import Inputs
|
||||
from app.analysis.probe import MAX_ROUNDS, Book, fetch
|
||||
|
||||
op, mode = _op_mode(request)
|
||||
raw = {k: v for k, v in (request.get("params") or {}).items() if k not in DERIVED_PARAMS}
|
||||
full = effective_settings(settings)
|
||||
prober = context.get("probe")
|
||||
today = context.get("today") or date.today()
|
||||
if isinstance(today, datetime):
|
||||
today = today.date()
|
||||
|
||||
def dialect(name: str, snap: Mapping[str, Any]) -> str | None:
|
||||
given = snap.get("dialect")
|
||||
if isinstance(given, str) and given:
|
||||
return given
|
||||
lookup = getattr(prober, "dialect", None)
|
||||
return lookup(str(snap.get("source") or "")) if lookup is not None else None
|
||||
|
||||
known: dict[tuple[str, ...], Any] = {}
|
||||
for rounds in range(MAX_ROUNDS + 1):
|
||||
book = Book(known)
|
||||
env = _env(op, mode, full, Inputs(resolve, dialect), Facts(catalogs=context.get("catalogs")), book, today,
|
||||
parents, request.get("hypothesis"))
|
||||
result = _compute(op, mode, raw, env)
|
||||
missing = book.missing
|
||||
if not missing or rounds == MAX_ROUNDS:
|
||||
return result
|
||||
known.update(await fetch(book, context))
|
||||
return result # pragma: no cover - 循环里一定返回
|
||||
|
||||
|
||||
def recompute(result: AnalysisResult, load: Callable[[str], InputSnapshot | None], *,
|
||||
load_result: Callable[[str], AnalysisResult | None] | None = None) -> AnalysisResult:
|
||||
"""复算：load 按 sha256 取回输入快照，load_result 按 sha256 取回 parent 那份结论。"""
|
||||
raise NotImplementedError("analysis.recompute：第 1 波 W1-a 实现")
|
||||
"""复算：load 按 sha256 取回输入快照，load_result 按 sha256 取回 parent 那份结论。用结论里记下的 params（含运行当天、
|
||||
目录事实）、settings、probes，不现查、不读目录。取不回输入时抛 CallError（code input_missing）；方法升版后调用不再成立
|
||||
时同样抛 CallError（照 run 的说法）。和封存的那份用 contract.same 比。"""
|
||||
from app.analysis.facts import Facts
|
||||
from app.analysis.inputs import _CELL, Inputs
|
||||
from app.analysis.probe import Book
|
||||
|
||||
op, mode = _op_mode(result)
|
||||
params = dict(result.get("params") or {})
|
||||
settings = effective_settings(result.get("settings"))
|
||||
names: dict[str, tuple[str | None, str | None, list[str]]] = {}
|
||||
parents_at: dict[str, str] = {}
|
||||
for item in result.get("inputs") or []:
|
||||
ref = str(item.get("ref") or "")
|
||||
if item.get("role") == "parent":
|
||||
parents_at[ref] = str(item.get("artifact") or "")
|
||||
continue
|
||||
cell = _CELL.match(ref) if "row" in item else None
|
||||
name = cell.group("name") if cell else ref
|
||||
names.setdefault(name, (item.get("artifact"), item.get("dialect"), list(item.get("masked") or [])))
|
||||
|
||||
def resolve(name: str) -> InputSnapshot | None:
|
||||
artifact, _, masked = names.get(name, (None, None, []))
|
||||
found = load(artifact) if artifact else None
|
||||
if not isinstance(found, Mapping):
|
||||
raise CallError("input_missing", f"复算取不回 {name} 的查询快照")
|
||||
return {**found, "artifact": artifact, "masked": masked} # type: ignore[typeddict-item]
|
||||
|
||||
def dialect(name: str, snap: Mapping[str, Any]) -> str | None:
|
||||
return names.get(name, (None, None, []))[1]
|
||||
|
||||
def parents(name: str) -> AnalysisResult | None:
|
||||
artifact = parents_at.get(name)
|
||||
found = load_result(artifact) if artifact and load_result is not None else None
|
||||
if not isinstance(found, Mapping):
|
||||
raise CallError("input_missing", f"复算取不回上一层的分析结论 {name}")
|
||||
return found # type: ignore[return-value]
|
||||
|
||||
today = date.fromisoformat(params["today"]) if isinstance(params.get("today"), str) else None
|
||||
env = _env(op, mode, settings, Inputs(resolve, dialect), Facts(recorded=params.get("catalog") or {}),
|
||||
Book(result.get("probes") or []), today, parents, result.get("hypothesis"))
|
||||
raw = {k: v for k, v in params.items() if k not in ("catalog", "today", "depth")}
|
||||
return _compute(op, mode, raw, env)
|
||||
|
||||
|
||||
__all__ = ["ParentResolver", "Resolver", "recompute", "run"]
|
||||
def exhausted(settings: Mapping[str, Any]) -> CallError:
|
||||
"""本运行分析次数用完（max_calls）的调用错误：入口数着次数，到了上限交回它，不再算。"""
|
||||
limit = int(effective_settings(settings)["max_calls"])
|
||||
return CallError("max_calls", f"这次运行的分析次数已经用完（最多 {limit} 次）：用已有的分析结论写，不再新做分析")
|
||||
|
||||
|
||||
def make_prober(*args: Any, **kwargs: Any) -> Any:
|
||||
"""见 app/analysis/probe.make_prober。"""
|
||||
from app.analysis.probe import make_prober as make
|
||||
|
||||
return make(*args, **kwargs)
|
||||
|
||||
|
||||
def comparable_reasons(result: Mapping[str, Any]) -> list[str]:
|
||||
"""可比性结论里 fail、warn 的原因，一句一个（口径卡指标的 comparability.reasons）。"""
|
||||
from app.analysis.comparable import reasons
|
||||
|
||||
return reasons(result)
|
||||
|
||||
|
||||
__all__ = ["ParentResolver", "Resolver", "comparable_reasons", "exhausted", "make_prober", "recompute", "run"]
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
"""算子共用的：一次计算的环境（输入、目录事实、现查簿、阈值）、结论的组装、几样参数的规范化。
|
||||
|
||||
每个算子是 (raw params, Env) → AnalysisResult 的纯函数：只经 Env 取输入快照（Inputs）、目录事实（Facts）、现查结果
|
||||
（Book），阈值只来自 Env.settings。run 和 recompute 只是装 Env 的方式不同（见 app/analysis/__init__.py）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis.contract import METHODS, AnalysisResult, CallError
|
||||
from app.analysis.facts import Facts
|
||||
from app.analysis.inputs import Inputs
|
||||
from app.analysis.numbers import to_fraction
|
||||
from app.analysis.probe import Book
|
||||
|
||||
#: 结论的 params 里由系统填、不收调用方的几项：目录事实、运行当天、下钻的层数
|
||||
DERIVED_PARAMS = frozenset({"catalog", "today", "depth"})
|
||||
|
||||
|
||||
@dataclass
|
||||
class Env:
|
||||
op: str
|
||||
mode: str | None
|
||||
settings: Mapping[str, Any]
|
||||
inputs: Inputs
|
||||
facts: Facts
|
||||
book: Book
|
||||
today: date | None
|
||||
parents: Callable[[str], Mapping[str, Any] | None] | None
|
||||
hypothesis: str | None
|
||||
|
||||
def threshold(self, key: str) -> Fraction:
|
||||
value = to_fraction(self.settings[key])
|
||||
assert value is not None, key
|
||||
return value
|
||||
|
||||
|
||||
def label_for(env: Env, raw: Any, snap: Any, column: str | None, *, default: str | None = None) -> str:
|
||||
"""判定语里的度量叫法：agent 给的 label → 目录里的中文名 → default → 列名，依次取第一个消毒后不为空的（agent 给了能用的
|
||||
就不去查目录）。"""
|
||||
if ph.clean_label(raw):
|
||||
return ph.clean_label(raw)
|
||||
catalog = env.facts.label(snap, column) if column else None
|
||||
return ph.pick_label(catalog, default, column)
|
||||
|
||||
|
||||
def column_label(env: Env, snap: Any, column: str) -> str:
|
||||
"""维度、因子这类列的叫法：目录里的中文名，没有就是列名（都消毒）。"""
|
||||
return ph.pick_label(env.facts.label(snap, column), column)
|
||||
|
||||
|
||||
def today_param(env: Env) -> str:
|
||||
"""要判本期完不完整的算子把运行当天记进 params（复算用记下的那一天）。"""
|
||||
day = env.today or date.today()
|
||||
env.today = day
|
||||
return day.isoformat()
|
||||
|
||||
|
||||
def build(env: Env, params: dict[str, Any], used: Sequence[str], *, status: str = "ok",
|
||||
reason_code: str | None = None, reason: str | None = None, table: Mapping[str, Any] | None = None,
|
||||
values: Mapping[str, Any] | None = None, verdict: Mapping[str, Any] | None = None, criterion: str,
|
||||
phrase: str, caveats: Sequence[Mapping[str, str]] = ()) -> AnalysisResult:
|
||||
"""组装一份结论：形状严格按 contract.AnalysisResult，params 补上查过的目录事实。"""
|
||||
record = env.facts.record()
|
||||
out_params = {k: v for k, v in params.items() if v is not None}
|
||||
if record:
|
||||
out_params["catalog"] = record
|
||||
return {
|
||||
"op": env.op,
|
||||
"mode": env.mode,
|
||||
"method": METHODS[(env.op, env.mode)],
|
||||
"params": out_params,
|
||||
"settings": {k: env.settings[k] for k in used},
|
||||
"inputs": list(env.inputs.records),
|
||||
"probes": env.book.used,
|
||||
"status": status, # type: ignore[typeddict-item]
|
||||
"reason_code": reason_code,
|
||||
"reason": reason,
|
||||
"table": dict(table) if table is not None else None, # type: ignore[typeddict-item]
|
||||
"values": dict(values or {}),
|
||||
"verdict": dict(verdict) if verdict is not None else None, # type: ignore[typeddict-item]
|
||||
"criterion": criterion,
|
||||
"phrase": phrase,
|
||||
"caveats": [dict(c) for c in caveats], # type: ignore[misc]
|
||||
"hypothesis": env.hypothesis,
|
||||
}
|
||||
|
||||
|
||||
def refuse(env: Env, params: dict[str, Any], used: Sequence[str], code: str, reason: str, *,
|
||||
criterion: str, caveats: Sequence[Mapping[str, str]] = ()) -> AnalysisResult:
|
||||
"""算不了（status refused）：判定语是「{算子的叫法}没有做成：{原因}」。"""
|
||||
return build(env, params, used, status="refused", reason_code=code, reason=reason, criterion=criterion,
|
||||
phrase=ph.refused(env.op, reason), caveats=caveats)
|
||||
|
||||
|
||||
def need(raw: Mapping[str, Any], key: str, what: str) -> Any:
|
||||
"""必填的参数；没写是调用错误。"""
|
||||
value = raw.get(key)
|
||||
if value is None or (isinstance(value, str) and not value.strip()) or value == []:
|
||||
raise CallError("missing_param", f"没写{what}")
|
||||
return value
|
||||
|
||||
|
||||
def written_places(raw: Any) -> int:
|
||||
"""快照里一格写到第几位小数（headline 是取整过的，按它的书写精度容差）：数字文本按字面，浮点按最短写法，整数为 0。"""
|
||||
if isinstance(raw, bool) or raw is None:
|
||||
return 0
|
||||
text = repr(raw) if isinstance(raw, float) else str(raw).strip()
|
||||
if "e" in text.lower():
|
||||
return 0
|
||||
return len(text.split(".", 1)[1]) if "." in text else 0
|
||||
|
||||
|
||||
def closes(total: Fraction, cell: Fraction, raw: Any) -> bool:
|
||||
"""各段之和（或合起来的比率）和总数那一格对得上：十进制逐位相等；总数取整过的，差在它书写精度的半个末位以内也算；浮点
|
||||
算出来的差在最后几位的也算（相对误差 1e-9 以内，同 engine/compare._equal）。"""
|
||||
if total == cell:
|
||||
return True
|
||||
gap = abs(total - cell)
|
||||
if gap <= Fraction(1, 2) / Fraction(10) ** written_places(raw):
|
||||
return True
|
||||
scale = max(abs(total), abs(cell))
|
||||
return scale != 0 and gap <= scale * Fraction(1, 10 ** 9)
|
||||
|
||||
|
||||
def sign_word(value: Fraction | float | int) -> str | None:
|
||||
return "up" if value > 0 else "down" if value < 0 else None
|
||||
|
||||
|
||||
__all__ = ["DERIVED_PARAMS", "Env", "build", "closes", "column_label", "label_for", "need", "refuse", "sign_word",
|
||||
"today_param", "written_places"]
|
||||
@@ -0,0 +1,540 @@
|
||||
"""可比性检查（细案 01 §6）：比两期之前先确认能比。三处用到：agent 单独调用（analysis_comparable）；变化拆解给了两份快照时
|
||||
先做；口径卡跨两份查询算环比、同比时自动做（入口经 run 调）。波动基线、趋势判本期完整用这里的 span_coverage。
|
||||
|
||||
五项检查，每项 pass / warn / fail / unknown（unknown 只写「核对不了」，不判不可比）：
|
||||
|
||||
- 同口径：两份 SQL 一字不差 → pass；规范化（data/sql_semantics.normalize_sql）后只差字面量：把时间条件的界（data/coverage
|
||||
认出的时间谓词的界，整个表达式）和像日期的字面量换成同一个占位再比，一样就是只差日期 → pass，不一样就是比的两段不是
|
||||
两期 → fail「两边除了日期，筛选的还不一样」；规范形式不同交给口径账本的 explain_sql_diff，等价 → pass，只差时间范围、排序
|
||||
→ pass，否则 fail，写出差在哪些方面（只写方面，不写 SQL 原文里的值）。两份来自不同的数据源 fail。同一份快照里按期次分组
|
||||
的天然同口径。
|
||||
- 窗口长度：用 data/coverage 的 filters 取业务日期列（目录的 business_date；没写就取唯一那一列时间谓词）上的区间，界是字面
|
||||
量的直接解析，按运行当天现算的在库里算（Book 的 bound，记进 probes）；算两边天数。整月对整月、整季对整季算一样长。
|
||||
不相等：度量是流量 fail，比率、存量、没标的 warn。按期次分组的：两期粒度相同，一样长。
|
||||
- 整期：窗口（或这一期）的终点和表里这一列的实际最大值（Book 的 range：现查，查不成退回剖析事实）、运行当天比；数据从
|
||||
窗口起点之后才开始的也算不完整。不完整：流量 fail（「本期数据只到 2026-10-15，还差 3 天」），其余 warn；表的范围查不到时
|
||||
只能按运行当天判，判不出的 unknown。
|
||||
- 星期构成：按天的窗口、天数不是 7 的倍数时数两边的周六、周日天数，不同 warn（「两段的周末天数不同：4 天对 6 天」）。整月、
|
||||
整季、按周的不看。
|
||||
- 数据问题：两边查询结果上「查询区间里这张表没有数据」（快照的 no_data）fail，SQL 检查的错误级结果（快照的 checks，level
|
||||
error）warn。
|
||||
|
||||
判定：有 fail → incomparable；只有 warn → conditional；其余 comparable（有核对不了的项照实写在判定语后面）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, timedelta
|
||||
from typing import Any
|
||||
|
||||
from app.analysis import periods as pd
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis.facts import Facts, date_source
|
||||
from app.analysis.inputs import Snap
|
||||
from app.analysis.probe import Book
|
||||
from app.data.names import name_key
|
||||
|
||||
CHECKS = ("same_query", "window", "complete", "weekdays", "data")
|
||||
CHECK_NAMES = {"same_query": "同口径", "window": "窗口长度", "complete": "整期", "weekdays": "星期构成",
|
||||
"data": "数据问题"}
|
||||
STATUS_NAMES = {"pass": "通过", "warn": "提醒", "fail": "不通过", "unknown": "核对不了"}
|
||||
#: explain_sql_diff 的方面 → 叫法
|
||||
_ASPECTS = {"time": "时间列", "time_range": "时间范围", "filter": "筛选条件", "aggregate": "聚合方式",
|
||||
"expression": "选出来的列", "join": "关联", "tables": "查的表", "set_op": "合并方式", "group_by": "分组",
|
||||
"having": "分组后的筛选", "limit": "取哪几行", "order": "排序", "parse": "写法", "other": "写法"}
|
||||
#: 只差这几个方面的两份 SQL 仍是同一个量的两期
|
||||
_SAME_ASPECTS = frozenset({"time_range", "order"})
|
||||
#: 一项 SQL 检查的说明最多引多少字
|
||||
_CHECK_TEXT_MAX = 80
|
||||
|
||||
|
||||
@dataclass
|
||||
class Check:
|
||||
id: str
|
||||
status: str = "unknown"
|
||||
reasons: list[str] = field(default_factory=list)
|
||||
base: str | None = None
|
||||
target: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Span:
|
||||
"""一边的时间范围：start / end 是窗口（或这一期）的头尾（含）；table / column 是这段时间按的那一列（探查表的范围用）；
|
||||
sql_window 是 SQL 时间条件在这一列上的区间（按期次分组时看它有没有截掉这一期）。"""
|
||||
|
||||
label: str
|
||||
snap: Snap
|
||||
start: date | None = None
|
||||
end: date | None = None
|
||||
table: str | None = None
|
||||
column: str | None = None
|
||||
sql_window: Any = None
|
||||
unit: str | None = None # 这一段是不是整月、整季、整年（calendar_unit），按期次时是粒度
|
||||
|
||||
|
||||
@dataclass
|
||||
class Outcome:
|
||||
checks: list[Check]
|
||||
verdict: str
|
||||
reasons: list[str]
|
||||
unknown: list[str]
|
||||
phrase: str
|
||||
structure: str | None # 同口径不通过的原因（变化拆解据此不拆）
|
||||
base: Span | None = None
|
||||
target: Span | None = None
|
||||
|
||||
def check(self, cid: str) -> Check:
|
||||
return next(c for c in self.checks if c.id == cid)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 同口径
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None:
|
||||
"""时间条件的界、像日期的字面量换成同一个占位之后的规范形式（文本 + 剩下的字面量）。解析不了为 None。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.data import coverage
|
||||
from app.data.sql_semantics import normalize_sql
|
||||
|
||||
tree = coverage.parse_sql(sql, read)
|
||||
if tree is None:
|
||||
return None
|
||||
for flt in coverage.filters(tree):
|
||||
for _, node, _ in flt.conds:
|
||||
if node.parent is not None:
|
||||
node.replace(exp.Literal.string("__day__"))
|
||||
for lit in list(tree.find_all(exp.Literal)):
|
||||
if coverage.literal_bound(lit)[1] is not None:
|
||||
lit.replace(exp.Literal.string("__day__"))
|
||||
normalized = normalize_sql(tree.sql(dialect=read), read)
|
||||
if normalized.parse_error:
|
||||
return None
|
||||
return normalized.text, normalized.literals
|
||||
|
||||
|
||||
def same_query(base: Snap, target: Snap) -> Check:
|
||||
from app.data.sql_semantics import explain_sql_diff, normalize_sql
|
||||
|
||||
check = Check("same_query")
|
||||
if base.source != target.source:
|
||||
check.status, check.reasons = "fail", ["两份查询来自不同的数据源"]
|
||||
return check
|
||||
a, b = base.sql.strip(), target.sql.strip()
|
||||
if a == b:
|
||||
check.status = "pass"
|
||||
return check
|
||||
read = base.dialect or target.dialect
|
||||
na, nb = normalize_sql(a, read), normalize_sql(b, read)
|
||||
if na.parse_error or nb.parse_error:
|
||||
check.reasons = ["SQL 解析不了"]
|
||||
return check
|
||||
if na.text == nb.text:
|
||||
ua, ub = _undated(a, read), _undated(b, read)
|
||||
if ua is None or ub is None:
|
||||
return check
|
||||
if ua == ub:
|
||||
check.status = "pass"
|
||||
else:
|
||||
check.status, check.reasons = "fail", ["两边除了日期，筛选的还不一样"]
|
||||
return check
|
||||
try:
|
||||
diff = explain_sql_diff(a, b, read)
|
||||
except Exception: # noqa: BLE001 - 比不了就当核对不了
|
||||
return check
|
||||
aspects = [d.aspect for d in diff.items]
|
||||
if diff.equivalent or (aspects and set(aspects) <= _SAME_ASPECTS):
|
||||
check.status = "pass"
|
||||
return check
|
||||
names = list(dict.fromkeys(_ASPECTS.get(x, "写法") for x in aspects if x not in _SAME_ASPECTS)) or ["写法"]
|
||||
check.status, check.reasons = "fail", [f"两份查询的口径结构不同（差在{'、'.join(names)}）"]
|
||||
return check
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 窗口
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _bound_days(snap: Snap, conds: Sequence[tuple[str, Any, bool]], book: Book) -> dict[int, Any]:
|
||||
from app.data import coverage
|
||||
|
||||
days: dict[int, Any] = {}
|
||||
for _, node, _ in conds:
|
||||
literal, day = coverage.literal_bound(node)
|
||||
if literal:
|
||||
days[id(node)] = day
|
||||
continue
|
||||
try:
|
||||
text = node.sql(dialect=snap.dialect)
|
||||
except Exception: # noqa: BLE001
|
||||
days[id(node)] = None
|
||||
continue
|
||||
probe = book.bound(snap.source, text)
|
||||
days[id(node)] = coverage.as_day(probe.get("value")) if probe else None
|
||||
return days
|
||||
|
||||
|
||||
def _filters_on(snap: Snap, measure: str | None, facts: Facts,
|
||||
column: tuple[str, str] | None = None) -> tuple[Any, str | None]:
|
||||
"""SQL（收窄到度量那一项）里按业务日期列（或唯一那一列时间谓词，或给定的那一列）的时间条件：(那段条件, 认不出的原因)。"""
|
||||
from app.data import coverage
|
||||
|
||||
tree = coverage.parse_sql(snap.sql, snap.dialect)
|
||||
if tree is None:
|
||||
return None, "SQL 解析不了"
|
||||
narrowed = coverage.narrowed(tree, measure, snap.dialect) if measure else tree
|
||||
flts = coverage.filters(narrowed)
|
||||
groups: dict[tuple[str, str], list[Any]] = {}
|
||||
for flt in flts:
|
||||
groups.setdefault((name_key(flt.table_name), name_key(flt.column_name)), []).append(flt)
|
||||
if column is not None:
|
||||
wanted = (name_key(column[0]), name_key(column[1]))
|
||||
wanted_short = (name_key(column[0].rsplit(".", 1)[-1]), wanted[1])
|
||||
chosen = groups.get(wanted) or next((v for k, v in groups.items()
|
||||
if (k[0].rsplit(".", 1)[-1], k[1]) == wanted_short), None)
|
||||
if chosen is None:
|
||||
return None, None
|
||||
else:
|
||||
if not groups:
|
||||
return None, "SQL 里没有时间条件"
|
||||
dated = [v for v in groups.values()
|
||||
if (bd := facts.business_date(snap.source, v[0].table_name)) and name_key(bd) == name_key(v[0].column_name)]
|
||||
if len(dated) == 1:
|
||||
chosen = dated[0]
|
||||
elif len(groups) == 1:
|
||||
chosen = next(iter(groups.values()))
|
||||
else:
|
||||
return None, "时间条件不止一列，认不出按哪一列算"
|
||||
if len({f.key for f in chosen}) > 1:
|
||||
return None, "这一列上有好几段时间条件"
|
||||
return chosen[0], None
|
||||
|
||||
|
||||
def sql_span(label: str, snap: Snap, measure: str | None, facts: Facts, book: Book) -> tuple[Span, str | None]:
|
||||
"""一份查询的时间窗口（两份快照比两期时）。"""
|
||||
from app.data import coverage
|
||||
|
||||
span = Span(label, snap)
|
||||
flt, why = _filters_on(snap, measure, facts)
|
||||
if flt is None:
|
||||
return span, why
|
||||
if flt.column_name.lower() in snap.masked:
|
||||
return span, "时间列是遮罩列" # 遮罩列不探表的范围
|
||||
span.table, span.column = flt.table_name, flt.column_name
|
||||
window = coverage.window_of(flt.conds, _bound_days(snap, flt.conds, book))
|
||||
if window is None or window.start is None or window.end is None:
|
||||
return span, "时间窗口没有两头"
|
||||
span.start, span.end = window.start, window.end
|
||||
span.unit = pd.calendar_unit(window.start, window.end)
|
||||
return span, None
|
||||
|
||||
|
||||
def period_span(label: str, snap: Snap, period: Any, item: pd.Period, grain: str, measure: str | None,
|
||||
facts: Facts, book: Book) -> Span:
|
||||
"""按期次分组的一份快照里的一期：这一期覆盖的日子，加上 SQL 在期次那一列上的时间条件。"""
|
||||
from app.data import coverage
|
||||
|
||||
span = Span(label, snap, item.start, item.end, unit=grain)
|
||||
found = date_source(snap.sql, period)
|
||||
if found is None:
|
||||
return span
|
||||
span.table, span.column = found
|
||||
if span.column.lower() in snap.masked:
|
||||
span.table = span.column = None
|
||||
return span
|
||||
flt, _ = _filters_on(snap, measure, facts, column=found)
|
||||
if flt is not None:
|
||||
span.sql_window = coverage.window_of(flt.conds, _bound_days(snap, flt.conds, book))
|
||||
return span
|
||||
|
||||
|
||||
def _range(span: Span, book: Book) -> tuple[bool, date | None, date | None, bool]:
|
||||
"""(查得到, 最早, 最晚, 一个值都没有)：表里这一列的实际范围。"""
|
||||
from app.data import coverage
|
||||
|
||||
if not (span.table and span.column):
|
||||
return False, None, None, False
|
||||
probe = book.range(span.snap.source, span.table, span.column)
|
||||
if probe is None:
|
||||
return False, None, None, False
|
||||
lo = coverage.as_day(probe.get("min")) if probe.get("min") else None
|
||||
hi = coverage.as_day(probe.get("max")) if probe.get("max") else None
|
||||
empty = probe.get("basis") == "live" and probe.get("min") is None and probe.get("max") is None
|
||||
return True, lo.day if lo else None, hi.day if hi else None, empty
|
||||
|
||||
|
||||
def span_coverage(span: Span, today: date, book: Book) -> tuple[str, str | None]:
|
||||
"""这一段（窗口或一期）的数据齐不齐：(pass / short / unknown, 不齐的那一句)。没过完（终点不早于运行当天：当天的数据还没
|
||||
到齐）、SQL 的时间条件截掉了一部分、表里的数据没到（最晚早于终点）或来得晚（最早晚于起点），都算不齐。"""
|
||||
if span.start is None or span.end is None:
|
||||
return "unknown", None
|
||||
lo, hi = span.start, span.end
|
||||
window = span.sql_window
|
||||
if window is not None:
|
||||
if window.start is not None:
|
||||
lo = max(lo, window.start)
|
||||
if window.end is not None:
|
||||
hi = min(hi, window.end)
|
||||
if span.end >= today:
|
||||
hi = min(hi, today - timedelta(days=1))
|
||||
known, dmin, dmax, empty = _range(span, book)
|
||||
if empty:
|
||||
return "short", f"{span.label}在 {span.start.isoformat()} 到 {span.end.isoformat()} 没有数据"
|
||||
if dmin is not None:
|
||||
lo = max(lo, dmin)
|
||||
if dmax is not None:
|
||||
hi = min(hi, dmax)
|
||||
if hi < lo:
|
||||
return "short", f"{span.label}在 {span.start.isoformat()} 到 {span.end.isoformat()} 没有数据"
|
||||
parts = []
|
||||
if hi < span.end:
|
||||
parts.append(f"{span.label}数据只到 {hi.isoformat()}，还差 {(span.end - hi).days} 天")
|
||||
if lo > span.start:
|
||||
parts.append(f"{span.label}数据从 {lo.isoformat()} 才开始，缺前 {(lo - span.start).days} 天")
|
||||
if parts:
|
||||
return "short", "，".join(parts)
|
||||
return ("pass", None) if known and dmax is not None else ("unknown", None)
|
||||
|
||||
|
||||
def _days(span: Span) -> int | None:
|
||||
return (span.end - span.start).days + 1 if span.start and span.end else None
|
||||
|
||||
|
||||
def _shown(span: Span) -> str | None:
|
||||
if span.start is None or span.end is None:
|
||||
return None
|
||||
if span.start == span.end:
|
||||
return f"{span.start.isoformat()}（1 天）"
|
||||
return f"{span.start.isoformat()} 到 {span.end.isoformat()}（{_days(span)} 天）"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 五项检查和判定
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _window_check(base: Span, target: Span, flow: bool, *, grouped: bool) -> Check:
|
||||
check = Check("window", base=_shown(base), target=_shown(target))
|
||||
a, b = _days(base), _days(target)
|
||||
if a is None or b is None:
|
||||
return check
|
||||
if a == b or grouped or (base.unit and base.unit == target.unit):
|
||||
check.status = "pass"
|
||||
return check
|
||||
check.status = "fail" if flow else "warn"
|
||||
check.reasons = [f"两段的天数不同：{a} 天对 {b} 天"]
|
||||
return check
|
||||
|
||||
|
||||
def _complete_check(sides: Sequence[Span], today: date, book: Book, flow: bool) -> Check:
|
||||
check = Check("complete")
|
||||
shorts, unknown = [], False
|
||||
for span in sides:
|
||||
status, reason = span_coverage(span, today, book)
|
||||
if status == "short":
|
||||
shorts.append(reason)
|
||||
elif status == "unknown":
|
||||
unknown = True
|
||||
if span.label == "基期":
|
||||
check.base = reason or STATUS_NAMES[status if status != "short" else "fail"]
|
||||
else:
|
||||
check.target = reason or STATUS_NAMES[status if status != "short" else "fail"]
|
||||
if shorts:
|
||||
check.status, check.reasons = ("fail" if flow else "warn"), [r for r in shorts if r]
|
||||
elif not unknown:
|
||||
check.status = "pass"
|
||||
return check
|
||||
|
||||
|
||||
def _weekday_check(base: Span, target: Span, *, grain: str | None) -> Check:
|
||||
check = Check("weekdays")
|
||||
a, b = _days(base), _days(target)
|
||||
if a is None or b is None:
|
||||
return check
|
||||
if grain in ("week", "month", "quarter", "seq") or (a % 7 == 0 and b % 7 == 0) or \
|
||||
(base.unit and base.unit == target.unit and base.unit in ("month", "quarter", "year")):
|
||||
check.status = "pass"
|
||||
return check
|
||||
wa, wb = pd.weekend_days(base.start, base.end), pd.weekend_days(target.start, target.end) # type: ignore[arg-type]
|
||||
check.base, check.target = f"周末 {wa} 天", f"周末 {wb} 天"
|
||||
if wa == wb:
|
||||
check.status = "pass"
|
||||
else:
|
||||
check.status, check.reasons = "warn", [f"两段的周末天数不同：{wa} 天对 {wb} 天"]
|
||||
return check
|
||||
|
||||
|
||||
def _data_check(sides: Sequence[tuple[str, Snap]], measure: str | None) -> Check:
|
||||
from app.data import coverage
|
||||
|
||||
check = Check("data", status="pass")
|
||||
fails, warns = [], []
|
||||
for label, snap in sides:
|
||||
found = coverage.no_data_of(snap.data, measure) if measure else \
|
||||
next(iter(snap.data.get("no_data") or []), None)
|
||||
if found:
|
||||
fails.append(f"{label}查询的区间里这张表没有数据")
|
||||
errors = [c for c in snap.data.get("checks") or [] if isinstance(c, Mapping) and c.get("level") == "error"]
|
||||
if errors:
|
||||
text = str(errors[0].get("message") or "").strip().replace("；", "，")
|
||||
if len(text) > _CHECK_TEXT_MAX:
|
||||
text = text[:_CHECK_TEXT_MAX - 1] + "…"
|
||||
warns.append(f"{label}查询的 SQL 检查有错误：{text}" if text else f"{label}查询的 SQL 检查有错误")
|
||||
if fails:
|
||||
check.status = "fail"
|
||||
elif warns:
|
||||
check.status = "warn"
|
||||
check.reasons = fails + warns
|
||||
return check
|
||||
|
||||
|
||||
def _decide(checks: list[Check], base: Span | None, target: Span | None) -> Outcome:
|
||||
fails = [r for c in checks if c.status == "fail" for r in c.reasons]
|
||||
warns = [r for c in checks if c.status == "warn" for r in c.reasons]
|
||||
unknown = [CHECK_NAMES[c.id] for c in checks if c.status == "unknown"]
|
||||
verdict = "incomparable" if fails else "conditional" if warns else "comparable"
|
||||
reasons = fails + warns
|
||||
structure = next((c.reasons[0] for c in checks if c.id == "same_query" and c.status == "fail" and c.reasons), None)
|
||||
return Outcome(checks, verdict, reasons, unknown, ph.comparable_phrase(verdict, reasons, unknown), structure,
|
||||
base, target)
|
||||
|
||||
|
||||
def compare_queries(base: Snap, target: Snap, measure: str | None, facts: Facts, book: Book, today: date) -> Outcome:
|
||||
"""两份快照比两期（本周一份、上周一份）。measure 是两边的度量列（收窄 SQL 找时间条件、认度量类型）；没有为 None。"""
|
||||
same = same_query(base, target)
|
||||
b_span, _ = sql_span("基期", base, measure, facts, book)
|
||||
t_span, _ = sql_span("本期", target, measure, facts, book)
|
||||
flow = bool(measure) and facts.measure(target, measure)[0] == "flow" # type: ignore[arg-type]
|
||||
checks = [same, _window_check(b_span, t_span, flow, grouped=False),
|
||||
_complete_check([b_span, t_span], today, book, flow), _weekday_check(b_span, t_span, grain=None),
|
||||
_data_check([("基期", base), ("本期", target)], measure)]
|
||||
return _decide(checks, b_span, t_span)
|
||||
|
||||
|
||||
def compare_periods(snap: Snap, period: str, base: pd.Period, target: pd.Period, grain: str, measure: str | None,
|
||||
facts: Facts, book: Book, today: date) -> Outcome:
|
||||
"""同一份快照里按期次分组的两期：天然同口径；窗口按期次的粒度；整期看这两期各自齐不齐。"""
|
||||
same = Check("same_query", status="pass")
|
||||
b_span = period_span("基期", snap, period, base, grain, measure, facts, book)
|
||||
t_span = period_span("本期", snap, period, target, grain, measure, facts, book)
|
||||
flow = bool(measure) and facts.measure(snap, measure)[0] == "flow" # type: ignore[arg-type]
|
||||
if grain == "seq":
|
||||
checks = [same, Check("window"), Check("complete"), Check("weekdays"), _data_check([("", snap)], measure)]
|
||||
else:
|
||||
checks = [same, _window_check(b_span, t_span, flow, grouped=True),
|
||||
_complete_check([b_span, t_span], today, book, flow),
|
||||
_weekday_check(b_span, t_span, grain=grain), _data_check([("", snap)], measure)]
|
||||
return _decide(checks, b_span, t_span)
|
||||
|
||||
|
||||
def table_of(outcome: Outcome) -> dict[str, Any]:
|
||||
"""检查表（结论的 table）：检查项、结果、基期、本期、说明。"""
|
||||
rows = [[CHECK_NAMES[c.id], STATUS_NAMES[c.status], c.base, c.target, "；".join(c.reasons) or None]
|
||||
for c in outcome.checks]
|
||||
return {"columns": ["check", "status", "base", "target", "note"],
|
||||
"labels": {"check": "检查项", "status": "结果", "base": "基期", "target": "本期", "note": "说明"},
|
||||
"rows": rows}
|
||||
|
||||
|
||||
def values_of(outcome: Outcome) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {c.id: c.status for c in outcome.checks}
|
||||
for side, span in (("base", outcome.base), ("target", outcome.target)):
|
||||
if span is not None and span.start and span.end:
|
||||
out[f"{side}_start"] = span.start.isoformat()
|
||||
out[f"{side}_end"] = span.end.isoformat()
|
||||
out[f"{side}_days"] = str(_days(span))
|
||||
out[f"{side}_weekend"] = str(pd.weekend_days(span.start, span.end))
|
||||
return out
|
||||
|
||||
|
||||
def reasons(result: Mapping[str, Any]) -> list[str]:
|
||||
"""一份可比性结论里 fail、warn 的原因，一句一个（口径卡指标的 comparability.reasons 用）。按检查表的说明拆开。"""
|
||||
out: list[str] = []
|
||||
table = result.get("table") or {}
|
||||
for row in table.get("rows") or []:
|
||||
if len(row) >= 5 and row[1] in (STATUS_NAMES["fail"], STATUS_NAMES["warn"]) and row[4]:
|
||||
out.extend(str(row[4]).split("；"))
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 算子：analysis_comparable
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def comparable(raw: Mapping[str, Any], env: Any) -> Any:
|
||||
"""可比性检查这个算子。参数：两份快照 base_query、target_query；或一份快照 query + period（期次列）+ base、target（期次
|
||||
列里的两个值）、可选 grain。都可以写 measure（度量列：收窄 SQL 找时间条件、按目录认它是不是流量；不写时只有一列数值列
|
||||
就用它）。"""
|
||||
from app.analysis.common import build, need, today_param
|
||||
from app.analysis.contract import CallError
|
||||
from app.analysis.inputs import same_value
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
two = raw.get("base_query") is not None or raw.get("target_query") is not None
|
||||
if two:
|
||||
base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号")
|
||||
target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号")
|
||||
if base is target:
|
||||
from app.analysis.decompose import SAME_QUERY
|
||||
|
||||
raise CallError("same_query", SAME_QUERY)
|
||||
params.update(base_query=base.ref, target_query=target.ref)
|
||||
else:
|
||||
base = target = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
|
||||
params["query"] = base.ref
|
||||
measure = target.column(raw["measure"], "度量列") if raw.get("measure") is not None else _only_number(target)
|
||||
if measure is not None and two:
|
||||
base.column(measure, "度量列")
|
||||
params["measure"] = measure
|
||||
params["today"] = today_param(env)
|
||||
if two:
|
||||
env.inputs.use(base, "base", [measure] if measure else [])
|
||||
env.inputs.use(target, "target", [measure] if measure else [])
|
||||
outcome = compare_queries(base, target, measure, env.facts, env.book, env.today)
|
||||
else:
|
||||
period = target.column(need(raw, "period", "期次列"), "期次列")
|
||||
grain = raw.get("grain")
|
||||
if grain is not None and grain not in (*pd.GRAINS, "seq"):
|
||||
raise CallError("bad_grain", "粒度只能写 day、week、month、quarter，或者不写（按期次值推断）")
|
||||
params.update(period=period, base=need(raw, "base", "基期（期次列里的值）"),
|
||||
target=need(raw, "target", "本期（期次列里的值）"))
|
||||
i = target.index(period)
|
||||
values = [row[i] if i < len(row) else None for row in target.rows]
|
||||
rows = {side: [r for r, v in enumerate(values) if same_value(params[side], v)] for side in ("base", "target")}
|
||||
for side, what in (("base", "基期"), ("target", "本期")):
|
||||
if not rows[side]:
|
||||
raise CallError("period_not_found", f"期次列「{period}」里没有「{params[side]}」：{what}要写 {target.ref} "
|
||||
"里有的期次值")
|
||||
env.inputs.use(target, "data", [period, *([measure] if measure else [])], rows["base"] + rows["target"])
|
||||
parsed = pd.parse(values, grain)
|
||||
if parsed is None:
|
||||
item_b = item_t = pd.Period(None, 0, None, None)
|
||||
grain = "seq"
|
||||
else:
|
||||
items, grain = parsed
|
||||
item_b, item_t = items[rows["base"][0]], items[rows["target"][0]]
|
||||
params["grain"] = grain
|
||||
outcome = compare_periods(target, period, item_b, item_t, grain, measure, env.facts, env.book, env.today)
|
||||
caveats = []
|
||||
window = outcome.check("window")
|
||||
if measure and window.status == "warn" and env.facts.measure(target, measure)[0] is None:
|
||||
caveats.append(ph.caveat("measure_unlabeled", ph.unlabeled_text(measure, "窗口不等只作提醒")))
|
||||
verdict = {"code": outcome.verdict, "subjects": [], "direction": None}
|
||||
return build(env, params, [], table=table_of(outcome), values=values_of(outcome), verdict=verdict,
|
||||
criterion=ph.CRITERION_COMPARABLE, phrase=outcome.phrase, caveats=caveats)
|
||||
|
||||
|
||||
def _only_number(snap: Snap) -> str | None:
|
||||
"""没写度量列时：快照里唯一一列（没遮罩的）数值列；没有或不止一列为 None。"""
|
||||
found = [c for c in snap.columns if c.lower() not in snap.masked and snap.kind(c) == "number"]
|
||||
return found[0] if len(found) == 1 else None
|
||||
|
||||
|
||||
__all__ = ["CHECKS", "CHECK_NAMES", "Check", "Outcome", "STATUS_NAMES", "Span", "comparable", "compare_periods",
|
||||
"compare_queries", "period_span", "reasons", "same_query", "span_coverage", "sql_span", "table_of",
|
||||
"values_of"]
|
||||
@@ -208,6 +208,7 @@ class AnalysisInput(TypedDict, total=False):
|
||||
column: str
|
||||
value: str | None
|
||||
dialect: str # 读这份快照的 SQL 用的方言（第 1 波 W1-a 加，可选）：复算按它读，界的写法才一样
|
||||
masked: list[str] # 这份快照当时不能当维度、度量的列（小写，只记列名；第 1 波 W1-a 加，可选）：复算按它认
|
||||
|
||||
|
||||
class Probe(TypedDict, total=False):
|
||||
|
||||
@@ -0,0 +1,732 @@
|
||||
"""变化拆解（细案 01 §2）：变化从哪来。三种模式，agent 按指标的性质选，算子核对选得对不对。
|
||||
|
||||
**参数**（AnalysisRequest.params；列名不分大小写，按快照认）：
|
||||
|
||||
- 一份快照：query（编号）、period（期次列）、base / target（期次列里的两个值）；或两份快照：base_query、target_query（各自
|
||||
只有维度列和度量列，先做可比性检查：口径结构不同的不拆，算不了 incomparable_sides；其余不可比的照样算，判定后附上
|
||||
可比性结论）。
|
||||
- additive：by（维度列，1–2 列）、measure（度量列）；可选 headline {base, target}（基期、本期总数各一格，如 Q1.r1.gmv，用来
|
||||
核对闭合）、parent {of: A3, key: [华东]}（下钻：这一层各段之和要等于 A3 里华东那一段）。
|
||||
- ratio：by、numerator（分子列）、denominator（分母列）；可选 headline {base, target}（整体比率的两格）、parent（上一层的比率
|
||||
拆解）。
|
||||
- factor：product（乘积列）、factors（因子列，2–4 个）；一份快照按 period / base / target 取两行，两份快照各一行。
|
||||
- 都可以写 label（度量的叫法，消毒后进判定语）。
|
||||
|
||||
**计算**全程 Fraction（快照里的数按字面换成精确的有理数）：各段之和逐位等于总变化；比率拆解的结构效应、自身效应按中点
|
||||
对称拆，Σmix + Σrate 逐位等于比率的变化；乘积拆解是 LMDI-I（对数），浮点，各因子贡献之和在浮点误差内等于总变化。判定、
|
||||
阈值、边界照细案 §2.1–§2.4。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
from app.analysis import comparable as cmp
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis.common import Env, build, closes, column_label, label_for, need, refuse, sign_word, today_param
|
||||
from app.analysis.contract import CallError
|
||||
from app.analysis.inputs import Snap, as_list, hashable, same_value
|
||||
from app.analysis.numbers import canon, to_fraction
|
||||
|
||||
#: 辛普森：分母够大的段里，按平均权重算有这么多的段比率和整体反着变
|
||||
SIMPSON_SHARE = Fraction(4, 5)
|
||||
#: 下钻、总数的写法（调用错误里教 agent 怎么写）
|
||||
PARENT_SHAPE = "下钻要写成 {of: A3, key: [华东]} 这样：of 是上一层变化拆解的编号，key 是那一段的维度值"
|
||||
HEADLINE_SHAPE = "总数要写成 {base: Q1.r1.gmv, target: Q1.r0.gmv} 这样：base 是基期总数那一格，target 是本期总数那一格"
|
||||
SAME_QUERY = "基期和本期是同一份查询：两期各给一份查询，或者给一份按期次分组的查询和期次列"
|
||||
#: 乘积闭合的最小相对误差
|
||||
_FACTOR_REL = Fraction(1, 10 ** 6)
|
||||
_NOT_ADDITIVE = {"ratio": "比率", "identifier": "标识", "status": "状态", "attribute": "属性"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Side:
|
||||
"""一期的各段：键 → 值（ratio 模式是 (分子, 分母)）；nulls 是这一期度量格为空的段；rows 是用到的行。"""
|
||||
|
||||
snap: Snap
|
||||
values: dict[tuple[Any, ...], Any] = field(default_factory=dict)
|
||||
nulls: list[tuple[Any, ...]] = field(default_factory=list)
|
||||
rows: list[int] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Segment:
|
||||
key: tuple[Any, ...]
|
||||
base: Any
|
||||
target: Any
|
||||
flags: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def _key_strings(key: Sequence[Any]) -> list[str]:
|
||||
return [ph.key_text([k]) for k in key]
|
||||
|
||||
|
||||
def _sides(raw: Mapping[str, Any], env: Env, roles: Sequence[tuple[str, str]], *, by_needed: bool = True
|
||||
) -> tuple[dict[str, Any], Snap, Snap, dict[str, str], list[str]]:
|
||||
"""一份或两份快照：(规范化的参数, 基期快照, 本期快照, {角色: 列名}, 维度列)。roles 是 [(参数名, 角色的叫法)]。"""
|
||||
params: dict[str, Any] = {}
|
||||
two = raw.get("base_query") is not None or raw.get("target_query") is not None
|
||||
if two:
|
||||
base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号")
|
||||
target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号")
|
||||
if base is target:
|
||||
raise CallError("same_query", SAME_QUERY)
|
||||
params.update(base_query=base.ref, target_query=target.ref)
|
||||
else:
|
||||
base = target = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
|
||||
params["query"] = base.ref
|
||||
by: list[str] = []
|
||||
if by_needed:
|
||||
wanted = as_list(raw.get("by"))
|
||||
if not 1 <= len(wanted) <= 2:
|
||||
raise CallError("bad_by", "维度列要写 1 到 2 列")
|
||||
by = [target.column(c, "维度列") for c in wanted]
|
||||
if two:
|
||||
for c in by:
|
||||
base.column(c, "维度列")
|
||||
params["by"] = by
|
||||
cols: dict[str, str] = {}
|
||||
for key, what in roles:
|
||||
if key == "factors":
|
||||
continue
|
||||
col = target.column(need(raw, key, what), what)
|
||||
if two:
|
||||
base.column(col, what)
|
||||
cols[key] = col
|
||||
params[key] = col
|
||||
if not two:
|
||||
params["period"] = target.column(need(raw, "period", "期次列"), "期次列")
|
||||
params["base"] = need(raw, "base", "基期（期次列里的值）")
|
||||
params["target"] = need(raw, "target", "本期（期次列里的值）")
|
||||
if same_value(params["base"], params["target"]):
|
||||
raise CallError("same_period", "基期和本期是同一期：写期次列里两个不同的值")
|
||||
used = [*cols.values(), *([params["period"]] if "period" in params else [])]
|
||||
if len(set(by)) != len(by) or set(by) & set(used) or len(set(used)) != len(used):
|
||||
raise CallError("bad_by", "维度列、期次列、度量列要是不同的列")
|
||||
return params, base, target, cols, by
|
||||
|
||||
|
||||
def _period_rows(snap: Snap, period: str, wanted: Any, what: str) -> list[int]:
|
||||
i = snap.index(period)
|
||||
rows = [r for r, row in enumerate(snap.rows) if i < len(row) and same_value(wanted, row[i])]
|
||||
if not rows:
|
||||
values = list(dict.fromkeys(ph.key_text([row[i]]) for row in snap.rows if i < len(row)))
|
||||
raise CallError("period_not_found", f"期次列「{period}」里没有「{wanted}」：{what}要写 {snap.ref} 里有的期次值，"
|
||||
f"比如 {'、'.join(values[:6])}")
|
||||
return rows
|
||||
|
||||
|
||||
def _rows_of(raw: Mapping[str, Any], params: dict[str, Any], base: Snap, target: Snap) -> tuple[list[int], list[int]]:
|
||||
if "period" in params:
|
||||
return (_period_rows(base, params["period"], params["base"], "基期"),
|
||||
_period_rows(target, params["period"], params["target"], "本期"))
|
||||
return list(range(len(base.rows))), list(range(len(target.rows)))
|
||||
|
||||
|
||||
def _collect(snap: Snap, rows: Sequence[int], by: Sequence[str], read: Any, labels: Sequence[str],
|
||||
period_label: str | None) -> Side:
|
||||
"""一期的各段：按维度键，同一键只许一行（多行是 SQL 没按这些维度汇总）。read(row) 读出这一行的值（None 是空）。"""
|
||||
side = Side(snap)
|
||||
seen: set[tuple[Any, ...]] = set()
|
||||
idx = [snap.index(c) for c in by]
|
||||
for r in rows:
|
||||
row = snap.rows[r]
|
||||
key = tuple(hashable(row[i] if i < len(row) else None) for i in idx)
|
||||
if key in seen:
|
||||
keys = "、".join([*labels, *([period_label] if period_label else [])])
|
||||
raise CallError("duplicate_rows", f"{snap.ref} 里同一个{'、'.join(labels)}在同一期有不止一行：在 SQL 里按 "
|
||||
f"{keys} 汇总，每段每期一行")
|
||||
seen.add(key)
|
||||
value = read(r)
|
||||
side.rows.append(r)
|
||||
if value is None:
|
||||
side.nulls.append(key)
|
||||
side.values[key] = value
|
||||
return side
|
||||
|
||||
|
||||
def _segments(base: Side, target: Side, zero: Any) -> list[Segment]:
|
||||
"""两期的各段，按先出现的顺序：缺的一边记 0，标「新出现 / 消失」；度量格为空的记 0，标「为空」。"""
|
||||
out: list[Segment] = []
|
||||
for key in dict.fromkeys([*base.values, *target.values]):
|
||||
flags = []
|
||||
if key not in base.values:
|
||||
flags.append("新出现")
|
||||
if key not in target.values:
|
||||
flags.append("消失")
|
||||
if key in base.nulls or key in target.nulls:
|
||||
flags.append("为空")
|
||||
b, t = base.values.get(key), target.values.get(key)
|
||||
out.append(Segment(key, zero if b is None else b, zero if t is None else t, flags))
|
||||
return out
|
||||
|
||||
|
||||
def _parent(raw: Mapping[str, Any], env: Env, mode: str, dims: int) -> tuple[dict[str, Any], list[Any], int]:
|
||||
"""下钻：上一层的结论和那一段。返回 (params 里的 parent, 那一段在上一层结果表里的那一行, 这一层的层数)。"""
|
||||
from app.core.artifact_store import canonical_json, content_hash
|
||||
|
||||
spec = raw.get("parent")
|
||||
name = spec.get("of") if isinstance(spec, Mapping) else None
|
||||
key = as_list(spec.get("key")) if isinstance(spec, Mapping) else []
|
||||
if not isinstance(name, str) or not name.strip() or not key:
|
||||
raise CallError("bad_parent", PARENT_SHAPE)
|
||||
name = name.strip()
|
||||
found = env.parents(name) if env.parents is not None else None
|
||||
if not isinstance(found, Mapping):
|
||||
raise CallError("parent_missing", f"没有叫「{name}」的分析结论：下钻写这次运行里变化拆解的编号（如 A3）")
|
||||
if found.get("op") != "decompose" or found.get("mode") != mode:
|
||||
raise CallError("parent_not_decompose", f"{name} 不是{'按维度' if mode == 'additive' else '比率'}的变化拆解，"
|
||||
"下钻要接在同一种拆解后面")
|
||||
if found.get("status") != "ok" or not isinstance(found.get("table"), Mapping):
|
||||
raise CallError("parent_refused", f"{name} 没有做成，不能在它下面下钻")
|
||||
depth = int((found.get("params") or {}).get("depth") or 0) + 1
|
||||
limit = int(env.settings["max_drill_depth"])
|
||||
if depth > limit:
|
||||
raise CallError("too_deep", f"已经下钻了 {depth - 1} 层，最多 {limit} 层：换一个维度从头拆")
|
||||
table = found["table"]
|
||||
fixed = _FIXED_ADDITIVE if mode == "additive" else _FIXED_RATIO
|
||||
columns = list(table.get("columns") or [])
|
||||
dim_count = len([c for c in columns if c not in fixed])
|
||||
if len(key) != dim_count:
|
||||
raise CallError("bad_parent", f"{name} 按 {dim_count} 列拆，下钻的那一段要写 {dim_count} 个维度值")
|
||||
hit = next((row for row in table.get("rows") or []
|
||||
if all(same_value(k, v) for k, v in zip(key, row[:dim_count]))), None)
|
||||
if hit is None:
|
||||
raise CallError("parent_key_missing", f"{name} 里没有「{ph.key_text(key)}」这一段：写 {name} 结果表里的段")
|
||||
env.inputs.parent(name, content_hash(canonical_json(found)))
|
||||
return {"of": name, "key": key}, dict(zip(columns, hit)), depth
|
||||
|
||||
|
||||
_FIXED_ADDITIVE = ("base", "target", "delta", "pct", "share", "flag")
|
||||
_FIXED_RATIO = ("n_base", "n_target", "d_base", "d_target", "w_base", "w_target", "r_base", "r_target", "mix", "rate",
|
||||
"total", "flag")
|
||||
|
||||
|
||||
def _headline(raw: Mapping[str, Any], env: Env) -> tuple[dict[str, str] | None, dict[str, tuple[Fraction, Any]]]:
|
||||
"""headline：基期、本期总数各一格 → ({base, target} 的写法, {base / target: (值, 原样)})。"""
|
||||
spec = raw.get("headline")
|
||||
if spec is None:
|
||||
return None, {}
|
||||
if isinstance(spec, (list, tuple)) and len(spec) == 2:
|
||||
spec = {"base": spec[0], "target": spec[1]}
|
||||
if not isinstance(spec, Mapping) or not spec.get("base") or not spec.get("target"):
|
||||
raise CallError("bad_headline", HEADLINE_SHAPE)
|
||||
out: dict[str, tuple[Fraction, Any]] = {}
|
||||
refs: dict[str, str] = {}
|
||||
for side, what in (("base", "基期总数"), ("target", "本期总数")):
|
||||
snap, row, col, value = env.inputs.cell(spec[side], f"headline_{side}", what)
|
||||
if value is None:
|
||||
raise CallError("headline_empty", f"{what}那一格（{snap.ref}.r{row}.{col}）是空的：换一格有值的")
|
||||
out[side] = (value, snap.cell(row, col))
|
||||
refs[side] = f"{snap.ref}.r{row}.{col}"
|
||||
return refs, out
|
||||
|
||||
|
||||
def _measure_guard(env: Env, snap: Snap, column: str, caveats: list[dict[str, str]], tail: str) -> None:
|
||||
"""additive 的度量列：目录标成（确认过的）比率、标识、状态、属性是调用错误；没标的、只是推断的照算，写附带说明。"""
|
||||
kind, confirmed = env.facts.measure(snap, column)
|
||||
if kind in _NOT_ADDITIVE and confirmed:
|
||||
if kind == "ratio":
|
||||
raise CallError("ratio_as_additive", f"「{column}」是比率（或平均值），不能按段相加：改用比率拆解，分子、分母各给一列")
|
||||
raise CallError("not_a_measure", f"「{column}」在目录里是{_NOT_ADDITIVE[kind]}，不是能相加的量：度量列要选金额、数量这类列")
|
||||
if kind is None:
|
||||
caveats.append(ph.caveat("measure_unlabeled", ph.unlabeled_text(column, tail)))
|
||||
elif kind in _NOT_ADDITIVE:
|
||||
caveats.append(ph.caveat("measure_unlabeled", ph.unconfirmed_text(column, kind, tail)))
|
||||
|
||||
|
||||
def _comparability(env: Env, params: dict[str, Any], base: Snap, target: Snap, measure: str | None
|
||||
) -> tuple[cmp.Outcome | None, str | None]:
|
||||
"""两份快照时先做可比性：(结论, 口径结构不同的原因)。"""
|
||||
if "base_query" not in params:
|
||||
return None, None
|
||||
params["today"] = today_param(env)
|
||||
outcome = cmp.compare_queries(base, target, measure, env.facts, env.book, env.today) # type: ignore[arg-type]
|
||||
return outcome, outcome.structure
|
||||
|
||||
|
||||
def _share_rank(segments: Sequence[Any], share: Any) -> list[Any]:
|
||||
"""按 share 从大到小（一样大的按先出现的）。"""
|
||||
return sorted(segments, key=lambda s: -share(s))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# additive
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def additive(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["main_share", "concentration", "offset_ratio", "flat_pct", "table_top_k"]
|
||||
params, bs, ts, cols, by = _sides(raw, env, [("measure", "度量列")])
|
||||
measure = cols["measure"]
|
||||
label = label_for(env, raw.get("label"), ts, measure)
|
||||
if raw.get("label") is not None:
|
||||
params["label"] = ph.clean_label(raw.get("label")) or None
|
||||
dims = [column_label(env, ts, c) for c in by]
|
||||
caveats: list[dict[str, str]] = []
|
||||
_measure_guard(env, ts, measure, caveats, "按可以按段相加处理")
|
||||
refs, heads = _headline(raw, env)
|
||||
if refs:
|
||||
params["headline"] = refs
|
||||
parent_row = None
|
||||
if raw.get("parent") is not None:
|
||||
params["parent"], parent_row, params["depth"] = _parent(raw, env, "additive", len(by))
|
||||
used.append("max_drill_depth")
|
||||
summed = not heads and parent_row is None
|
||||
criterion_args = {"summed": summed}
|
||||
for snap in dict.fromkeys([bs, ts]):
|
||||
if snap.truncated:
|
||||
env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [*by, measure])
|
||||
return refuse(env, params, used, "truncated",
|
||||
f"{snap.ref} 只取回了前 {len(snap.rows)} 行，各段之和不是完整的和",
|
||||
criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args))
|
||||
rows_b, rows_t = _rows_of(raw, params, bs, ts)
|
||||
period_label = column_label(env, ts, params["period"]) if "period" in params else None
|
||||
side_b = _collect(bs, rows_b, by, lambda r: bs.number(r, measure, "度量列"), dims, period_label)
|
||||
side_t = _collect(ts, rows_t, by, lambda r: ts.number(r, measure, "度量列"), dims, period_label)
|
||||
if bs is ts:
|
||||
env.inputs.use(ts, "data", [*by, params["period"], measure], side_b.rows + side_t.rows)
|
||||
else:
|
||||
env.inputs.use(bs, "base", [*by, measure], side_b.rows)
|
||||
env.inputs.use(ts, "target", [*by, measure], side_t.rows)
|
||||
outcome, structure = _comparability(env, params, bs, ts, measure)
|
||||
if structure is not None:
|
||||
return refuse(env, params, used, "incomparable_sides", f"{structure}，不是同一个量的两期，不拆",
|
||||
criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats)
|
||||
segs = _segments(side_b, side_t, Fraction(0))
|
||||
if len(segs) < 2:
|
||||
return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(segs)} 段，至少要 2 段才拆得开",
|
||||
criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats)
|
||||
return _additive_result(env, params, used, segs, by, dims, label, heads, parent_row, outcome, caveats, summed,
|
||||
side_b, side_t)
|
||||
|
||||
|
||||
def _additive_result(env: Env, params: dict[str, Any], used: list[str], segs: list[Segment], by: list[str],
|
||||
dims: list[str], label: str, heads: Mapping[str, tuple[Fraction, Any]],
|
||||
parent_row: Mapping[str, Any] | None, outcome: cmp.Outcome | None,
|
||||
caveats: list[dict[str, str]], summed: bool, side_b: Side, side_t: Side) -> Any:
|
||||
s = env.settings
|
||||
main_share, concentration = env.threshold("main_share"), env.threshold("concentration")
|
||||
offset_ratio, flat_pct = env.threshold("offset_ratio"), env.threshold("flat_pct")
|
||||
B = sum((x.base for x in segs), Fraction(0))
|
||||
T = sum((x.target for x in segs), Fraction(0))
|
||||
delta = T - B
|
||||
gaps: dict[str, Fraction] = {}
|
||||
for side, total in (("base", B), ("target", T)):
|
||||
if side in heads and not closes(total, heads[side][0], heads[side][1]):
|
||||
gaps[side] = heads[side][0] - total
|
||||
if parent_row is not None:
|
||||
expect = to_fraction(parent_row.get(side))
|
||||
if expect is None or not closes(total, expect, parent_row.get(side)):
|
||||
gaps.setdefault(side, (expect or Fraction(0)) - total)
|
||||
closed = not gaps
|
||||
deltas = {x.key: x.target - x.base for x in segs}
|
||||
shares = {k: d / delta for k, d in deltas.items()} if closed and delta != 0 else {}
|
||||
gross_up = sum((d for d in deltas.values() if d > 0), Fraction(0))
|
||||
gross_down = sum((d for d in deltas.values() if d < 0), Fraction(0))
|
||||
opposite = abs(gross_down) if delta > 0 else gross_up if delta < 0 else gross_up
|
||||
n = len(segs)
|
||||
k_max = max(2, math.ceil(n / 5))
|
||||
flat = delta == 0 or (B != 0 and abs(delta) / abs(B) < flat_pct)
|
||||
ranked = _share_rank(segs, lambda x: shares[x.key]) if shares else []
|
||||
top_k = top_k_share = None
|
||||
if ranked:
|
||||
running = Fraction(0)
|
||||
for k, seg in enumerate(ranked, start=1):
|
||||
running += shares[seg.key]
|
||||
if running >= concentration:
|
||||
top_k, top_k_share = k, running
|
||||
break
|
||||
if not closed:
|
||||
code, subjects = "unclosed", []
|
||||
elif flat:
|
||||
code, subjects = "flat", []
|
||||
elif shares[ranked[0].key] >= main_share:
|
||||
code, subjects = "driver", [ranked[0]]
|
||||
elif top_k is not None and 2 <= top_k <= k_max:
|
||||
code, subjects = "concentrated", ranked[:top_k]
|
||||
else:
|
||||
code, subjects = "dispersed", []
|
||||
offset = opposite > 0 and opposite >= offset_ratio * abs(delta)
|
||||
if offset:
|
||||
caveats.append(ph.caveat("offset", ph.offset_text(delta, gross_up, gross_down)))
|
||||
nulls = [x for x in segs if "为空" in x.flags]
|
||||
if nulls:
|
||||
caveats.append(ph.caveat("null_values", ph.null_segments_text([ph.key_text(x.key) for x in nulls], label)))
|
||||
if summed:
|
||||
caveats.append(ph.caveat("headline_basis", ph.HEADLINE_BASIS))
|
||||
if outcome is not None and outcome.verdict != "comparable":
|
||||
caveats.append(ph.caveat("comparability", outcome.phrase))
|
||||
top = ranked[0] if ranked else max(segs, key=lambda x: abs(deltas[x.key]))
|
||||
total_text = ph.change(delta, B)
|
||||
top_n = int(s["table_top_k"])
|
||||
if code == "driver":
|
||||
main = ph.decompose_driver(label, total_text, ph.key_text(top.key), ph.amount(deltas[top.key], signed=True),
|
||||
shares[top.key])
|
||||
elif code == "concentrated":
|
||||
main = ph.decompose_concentrated(label, total_text, ph.names([ph.key_text(x.key) for x in subjects], top_n),
|
||||
top_k_share)
|
||||
elif code == "dispersed":
|
||||
main = ph.decompose_dispersed(label, total_text, ph.dims_text(dims), main_share)
|
||||
elif code == "flat":
|
||||
ups = [x for x in segs if deltas[x.key] > 0]
|
||||
downs = [x for x in segs if deltas[x.key] < 0]
|
||||
pair = None
|
||||
if offset and ups and downs:
|
||||
up = max(ups, key=lambda x: deltas[x.key])
|
||||
down = min(downs, key=lambda x: deltas[x.key])
|
||||
pair = (ph.key_text(up.key), ph.amount(deltas[up.key], signed=True), ph.key_text(down.key),
|
||||
ph.amount(deltas[down.key], signed=True))
|
||||
main = ph.decompose_flat(label, delta / abs(B) if B != 0 else None, pair)
|
||||
else:
|
||||
gap = max(gaps.values(), key=abs)
|
||||
main = ph.decompose_unclosed(ph.dims_text(dims), gap, ratio=False)
|
||||
order = sorted(segs, key=lambda x: -abs(deltas[x.key]))
|
||||
rows = []
|
||||
for x in order:
|
||||
d = deltas[x.key]
|
||||
rows.append([*x.key, canon(x.base), canon(x.target), canon(d), canon(d / x.base) if x.base != 0 else None,
|
||||
canon(shares[x.key]) if shares else None, "、".join(x.flags) or None])
|
||||
table = {"columns": [*by, *_FIXED_ADDITIVE],
|
||||
"labels": {**dict(zip(by, dims)), "base": "基期", "target": "本期", "delta": "变化", "pct": "变化率",
|
||||
"share": "占变化的比重", "flag": "说明"},
|
||||
"rows": rows}
|
||||
values: dict[str, Any] = {
|
||||
"base_total": canon(B), "target_total": canon(T), "delta_total": canon(delta),
|
||||
"pct_total": canon(delta / abs(B)) if B != 0 else None, "top_key": ph.key_text(top.key),
|
||||
"top_share": canon(shares[top.key]) if shares else None, "top_k": str(top_k) if top_k is not None else None,
|
||||
"top_k_share": canon(top_k_share) if top_k_share is not None else None, "gross_up": canon(gross_up),
|
||||
"gross_down": canon(gross_down), "segments": str(n),
|
||||
}
|
||||
if gaps:
|
||||
values["gap"] = canon(max(gaps.values(), key=abs))
|
||||
verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects],
|
||||
"direction": sign_word(delta) if code in ("driver", "concentrated", "dispersed") else None}
|
||||
phrase = ph.with_caveats(main, caveats, skip=("offset",) if code == "flat" else ())
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict,
|
||||
criterion=ph.criterion_additive(s, top_k_max=k_max, summed=summed), phrase=phrase, caveats=caveats)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# ratio
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def ratio(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["main_share", "flat_pct", "min_denominator", "table_top_k"]
|
||||
params, bs, ts, cols, by = _sides(raw, env, [("numerator", "分子列"), ("denominator", "分母列")])
|
||||
num, den = cols["numerator"], cols["denominator"]
|
||||
num_label, den_label = column_label(env, ts, num), column_label(env, ts, den)
|
||||
label = ph.pick_label(raw.get("label"), ph.say(num_label, "与", den_label, "之比"))
|
||||
if raw.get("label") is not None:
|
||||
params["label"] = ph.clean_label(raw.get("label")) or None
|
||||
dims = [column_label(env, ts, c) for c in by]
|
||||
refs, heads = _headline(raw, env)
|
||||
if refs:
|
||||
params["headline"] = refs
|
||||
parent_row = None
|
||||
if raw.get("parent") is not None:
|
||||
params["parent"], parent_row, params["depth"] = _parent(raw, env, "ratio", len(by))
|
||||
used.append("max_drill_depth")
|
||||
criterion = ph.criterion_ratio(env.settings)
|
||||
for snap in dict.fromkeys([bs, ts]):
|
||||
if snap.truncated:
|
||||
env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [*by, num, den])
|
||||
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，各段合起来不是完整的数",
|
||||
criterion=criterion)
|
||||
rows_b, rows_t = _rows_of(raw, params, bs, ts)
|
||||
period_label = column_label(env, ts, params["period"]) if "period" in params else None
|
||||
|
||||
def reader(snap: Snap) -> Any:
|
||||
return lambda r: (snap.number(r, num, "分子列"), snap.number(r, den, "分母列"))
|
||||
|
||||
side_b = _collect(bs, rows_b, by, reader(bs), dims, period_label)
|
||||
side_t = _collect(ts, rows_t, by, reader(ts), dims, period_label)
|
||||
for side in (side_b, side_t):
|
||||
side.nulls = [k for k, v in side.values.items() if v[0] is None or v[1] is None]
|
||||
side.values = {k: (v[0] or Fraction(0), v[1] or Fraction(0)) for k, v in side.values.items()}
|
||||
if bs is ts:
|
||||
env.inputs.use(ts, "data", [*by, params["period"], num, den], side_b.rows + side_t.rows)
|
||||
else:
|
||||
env.inputs.use(bs, "base", [*by, num, den], side_b.rows)
|
||||
env.inputs.use(ts, "target", [*by, num, den], side_t.rows)
|
||||
outcome, structure = _comparability(env, params, bs, ts, den)
|
||||
if structure is not None:
|
||||
return refuse(env, params, used, "incomparable_sides", f"{structure}，不是同一个量的两期，不拆", criterion=criterion)
|
||||
segs = _segments(side_b, side_t, (Fraction(0), Fraction(0)))
|
||||
live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0]
|
||||
if len(live) < 2:
|
||||
return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(live)} 段，至少要 2 段才拆得开",
|
||||
criterion=criterion)
|
||||
for when, i in (("基期", 0), ("本期", 1)):
|
||||
if sum(((x.base, x.target)[i][1] for x in live), Fraction(0)) == 0:
|
||||
return refuse(env, params, used, "zero_base", ph.say(f"{when}的", den_label, "合计为 0，算不了比率"),
|
||||
criterion=criterion)
|
||||
for x in live:
|
||||
for (n_, d_), when in ((x.base, "基期"), (x.target, "本期")):
|
||||
if d_ == 0 and n_ != 0:
|
||||
return refuse(env, params, used, "zero_base",
|
||||
ph.say(ph.key_text(x.key), f"{when}的", den_label, "为 0 而", num_label,
|
||||
"不为 0，比率拆不了"), criterion=criterion)
|
||||
return _ratio_result(env, params, used, segs, by, dims, label, num_label, den_label, heads, parent_row, outcome,
|
||||
criterion)
|
||||
|
||||
|
||||
def ratio_effects(segs: Sequence[tuple[Fraction, Fraction, Fraction, Fraction]]
|
||||
) -> tuple[Fraction, Fraction, list[tuple[Fraction, ...]]]:
|
||||
"""比率拆解的计算（细案 §2.2，中点对称拆法）。segs 是各段 (分子_基期, 分母_基期, 分子_本期, 分母_本期)，两期分母都为 0
|
||||
的段先剔掉。交回 (R, R', 各段 (w, w', r, r', mix, rate))：Σmix + Σrate = R' − R 逐位成立（Σ(w' − w) = 0）。一边分母为 0
|
||||
的段：那一边权重 0、比率取另一边的（自身效应为 0，变化全记在结构上）。"""
|
||||
db = sum((s[1] for s in segs), Fraction(0))
|
||||
dt = sum((s[3] for s in segs), Fraction(0))
|
||||
R = sum((s[0] for s in segs), Fraction(0)) / db
|
||||
R2 = sum((s[2] for s in segs), Fraction(0)) / dt
|
||||
mid = (R + R2) / 2
|
||||
out = []
|
||||
for nb, d_b, nt, d_t in segs:
|
||||
w, w2 = d_b / db, d_t / dt
|
||||
r = nb / d_b if d_b != 0 else None
|
||||
r2 = nt / d_t if d_t != 0 else None
|
||||
r = r2 if r is None else r
|
||||
r2 = r if r2 is None else r2
|
||||
mix = (w2 - w) * ((r + r2) / 2 - mid) # type: ignore[operator]
|
||||
rate = (r2 - r) * (w + w2) / 2 # type: ignore[operator]
|
||||
out.append((w, w2, r, r2, mix, rate))
|
||||
return R, R2, out
|
||||
|
||||
|
||||
def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[Segment], by: list[str],
|
||||
dims: list[str], label: str, num_label: str, den_label: str,
|
||||
heads: Mapping[str, tuple[Fraction, Any]], parent_row: Mapping[str, Any] | None,
|
||||
outcome: cmp.Outcome | None, criterion: str) -> Any:
|
||||
main_share, flat_pct = env.threshold("main_share"), env.threshold("flat_pct")
|
||||
min_den = env.threshold("min_denominator")
|
||||
live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0]
|
||||
dropped = [x for x in segs if x not in live]
|
||||
R, R2, effects = ratio_effects([(x.base[0], x.base[1], x.target[0], x.target[1]) for x in live])
|
||||
delta = R2 - R
|
||||
by_key = {x.key: e for x, e in zip(live, effects)}
|
||||
gaps: dict[str, Fraction] = {}
|
||||
for side, total in (("base", R), ("target", R2)):
|
||||
if side in heads and not closes(total, heads[side][0], heads[side][1]):
|
||||
gaps[side] = heads[side][0] - total
|
||||
if parent_row is not None:
|
||||
sums = {"n_base": sum((x.base[0] for x in live), Fraction(0)), "d_base": sum((x.base[1] for x in live), Fraction(0)),
|
||||
"n_target": sum((x.target[0] for x in live), Fraction(0)),
|
||||
"d_target": sum((x.target[1] for x in live), Fraction(0))}
|
||||
for col, total in sums.items():
|
||||
expect = to_fraction(parent_row.get(col))
|
||||
if expect is None or not closes(total, expect, parent_row.get(col)):
|
||||
gaps.setdefault("base" if col.endswith("base") else "target", (expect or Fraction(0)) - total)
|
||||
closed = not gaps
|
||||
mix_total = sum((e[4] for e in effects), Fraction(0))
|
||||
rate_total = sum((e[5] for e in effects), Fraction(0))
|
||||
flat = delta == 0 or (R != 0 and abs(delta) / abs(R) < flat_pct)
|
||||
mix_share = mix_total / delta if delta != 0 else None
|
||||
rate_share = rate_total / delta if delta != 0 else None
|
||||
sign = 1 if delta >= 0 else -1
|
||||
if not closed:
|
||||
code, subjects = "unclosed", []
|
||||
elif flat:
|
||||
code, subjects = "flat", []
|
||||
elif mix_share >= main_share and rate_share < main_share: # type: ignore[operator]
|
||||
code = "mix_driven"
|
||||
subjects = [max(live, key=lambda x: sign * by_key[x.key][4])]
|
||||
elif rate_share >= main_share and mix_share < main_share: # type: ignore[operator]
|
||||
code = "rate_driven"
|
||||
subjects = [max(live, key=lambda x: sign * by_key[x.key][5])]
|
||||
else:
|
||||
code, subjects = "both", []
|
||||
proportion = all(0 <= v[0] <= v[1] for x in live for v in (x.base, x.target))
|
||||
small = [x for x in live if x.base[1] < min_den or x.target[1] < min_den]
|
||||
caveats: list[dict[str, str]] = []
|
||||
eligible = [x for x in live if x not in small]
|
||||
if delta != 0 and eligible:
|
||||
weight = sum(((by_key[x.key][0] + by_key[x.key][1]) / 2 for x in eligible), Fraction(0))
|
||||
against = sum(((by_key[x.key][0] + by_key[x.key][1]) / 2 for x in eligible
|
||||
if (by_key[x.key][3] - by_key[x.key][2]) * sign < 0), Fraction(0))
|
||||
if weight > 0 and against / weight >= SIMPSON_SHARE:
|
||||
star = max(live, key=lambda x: sign * by_key[x.key][4])
|
||||
caveats.append(ph.caveat("simpson", ph.simpson_text(label, delta > 0, ph.key_text(star.key),
|
||||
by_key[star.key][0], by_key[star.key][1])))
|
||||
if small:
|
||||
caveats.append(ph.caveat("small_denominator", ph.small_denominator_text(
|
||||
[ph.key_text(x.key) for x in small], den_label, int(env.settings["min_denominator"]))))
|
||||
nulls = [x for x in segs if "为空" in x.flags]
|
||||
if nulls:
|
||||
caveats.append(ph.caveat("null_values", ph.null_segments_text([ph.key_text(x.key) for x in nulls],
|
||||
ph.say(num_label, "或", den_label))))
|
||||
if outcome is not None and outcome.verdict != "comparable":
|
||||
caveats.append(ph.caveat("comparability", outcome.phrase))
|
||||
total_text = ph.ratio_change(R, R2, proportion=proportion)
|
||||
if code == "mix_driven":
|
||||
star = subjects[0]
|
||||
main = ph.ratio_mix(label, total_text, ph.key_text(star.key), by_key[star.key][0], by_key[star.key][1])
|
||||
elif code == "rate_driven":
|
||||
main = ph.ratio_rate(label, total_text, ph.key_text(subjects[0].key))
|
||||
elif code == "both":
|
||||
main = ph.ratio_both(label, total_text, mix_share, rate_share)
|
||||
elif code == "flat":
|
||||
main = ph.decompose_flat(label, delta / abs(R) if R != 0 else None, None)
|
||||
else:
|
||||
main = ph.decompose_unclosed(ph.dims_text(dims), max(gaps.values(), key=abs), ratio=True)
|
||||
rows = []
|
||||
for x in sorted(live, key=lambda x: -abs(by_key[x.key][4] + by_key[x.key][5])):
|
||||
w, w2, r, r2, mix, rate = by_key[x.key]
|
||||
flags = list(x.flags)
|
||||
if x.base[1] == 0:
|
||||
flags.append("基期分母为 0")
|
||||
if x.target[1] == 0:
|
||||
flags.append("本期分母为 0")
|
||||
if x in small:
|
||||
flags.append("分母小")
|
||||
rows.append([*x.key, canon(x.base[0]), canon(x.target[0]), canon(x.base[1]), canon(x.target[1]), canon(w),
|
||||
canon(w2), canon(r), canon(r2), canon(mix), canon(rate), canon(mix + rate),
|
||||
"、".join(dict.fromkeys(flags)) or None])
|
||||
for x in dropped:
|
||||
rows.append([*x.key, canon(x.base[0]), canon(x.target[0]), "0", "0", None, None, None, None, None, None, None,
|
||||
"两期分母都为 0，不计入"])
|
||||
table = {"columns": [*by, *_FIXED_RATIO],
|
||||
"labels": {**dict(zip(by, dims)), "n_base": ph.say("基期", num_label), "n_target": ph.say("本期", num_label),
|
||||
"d_base": ph.say("基期", den_label), "d_target": ph.say("本期", den_label),
|
||||
"w_base": "基期权重", "w_target": "本期权重", "r_base": "基期比率", "r_target": "本期比率",
|
||||
"mix": "结构效应", "rate": "自身效应", "total": "合计影响", "flag": "说明"},
|
||||
"rows": rows}
|
||||
values: dict[str, Any] = {
|
||||
"r_base_total": canon(R), "r_target_total": canon(R2), "delta": canon(delta), "mix_total": canon(mix_total),
|
||||
"rate_total": canon(rate_total), "mix_share": canon(mix_share), "rate_share": canon(rate_share),
|
||||
"segments": str(len(live)),
|
||||
}
|
||||
if gaps:
|
||||
values["gap"] = canon(max(gaps.values(), key=abs))
|
||||
verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects],
|
||||
"direction": sign_word(delta) if code in ("mix_driven", "rate_driven", "both") else None}
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
|
||||
phrase=ph.with_caveats(main, caveats), caveats=caveats)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# factor
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _half_ulp_ratio(raw: Any, value: Fraction) -> Fraction:
|
||||
from app.analysis.common import written_places
|
||||
|
||||
return Fraction(1, 2) / Fraction(10) ** written_places(raw) / abs(value) if value != 0 else Fraction(0)
|
||||
|
||||
|
||||
def lmdi(base: Sequence[float], target: Sequence[float], m_base: float, m_target: float) -> list[float]:
|
||||
"""LMDI-I：contrib_k = L(M', M) · ln(F_k' / F_k)，L(a, b) = (a − b) / (ln a − ln b)，L(a, a) = a。因子按书写精度取过整、
|
||||
乘积和乘积列差一点时，各因子按同一比例缩放到乘积恰好等于乘积列（每个因子的对数比各补上 (1/K)·差的对数），Σcontrib 在
|
||||
浮点误差内等于 M' − M。"""
|
||||
k = len(base)
|
||||
fix = (math.log(m_target / math.prod(target)) - math.log(m_base / math.prod(base))) / k
|
||||
weight = m_base if m_base == m_target else (m_target - m_base) / (math.log(m_target) - math.log(m_base))
|
||||
return [weight * (math.log(t / b) + fix) for b, t in zip(base, target)]
|
||||
|
||||
|
||||
def factor(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["main_share", "flat_pct"]
|
||||
params, bs, ts, cols, _ = _sides(raw, env, [("product", "乘积列")], by_needed=False)
|
||||
product = cols["product"]
|
||||
wanted = as_list(raw.get("factors"))
|
||||
if not 2 <= len(wanted) <= 4:
|
||||
raise CallError("bad_factors", "因子列要写 2 到 4 列")
|
||||
factors = [ts.column(c, "因子列") for c in wanted]
|
||||
if bs is not ts:
|
||||
for c in factors:
|
||||
bs.column(c, "因子列")
|
||||
if len(set(factors)) != len(factors) or product in factors:
|
||||
raise CallError("bad_factors", "因子列不能重复，也不能是乘积列本身")
|
||||
params["factors"] = factors
|
||||
label = label_for(env, raw.get("label"), ts, product)
|
||||
if raw.get("label") is not None:
|
||||
params["label"] = ph.clean_label(raw.get("label")) or None
|
||||
names = [column_label(env, ts, c) for c in factors]
|
||||
criterion = ph.criterion_factor(env.settings)
|
||||
for snap in dict.fromkeys([bs, ts]):
|
||||
if snap.truncated:
|
||||
env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [product, *factors])
|
||||
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，结果不完整",
|
||||
criterion=criterion)
|
||||
if "period" in params:
|
||||
rows_b = _period_rows(bs, params["period"], params["base"], "基期")
|
||||
rows_t = _period_rows(ts, params["period"], params["target"], "本期")
|
||||
else:
|
||||
rows_b, rows_t = list(range(len(bs.rows))), list(range(len(ts.rows)))
|
||||
for rows, snap, when in ((rows_b, bs, "基期"), (rows_t, ts, "本期")):
|
||||
if len(rows) != 1:
|
||||
raise CallError("duplicate_rows", f"{snap.ref} 里{when}有 {len(rows)} 行：乘积拆解每期要正好一行（在 SQL 里汇总成一行）")
|
||||
cells: dict[str, list[tuple[Fraction, Any]]] = {}
|
||||
for when, snap, r in (("base", bs, rows_b[0]), ("target", ts, rows_t[0])):
|
||||
row = []
|
||||
for col, what in ((product, "乘积列"), *((c, "因子列") for c in factors)):
|
||||
value = snap.number(r, col, what)
|
||||
if value is None:
|
||||
raise CallError("empty_cell", f"{snap.ref} 第 {r} 行的「{col}」是空的：乘积、因子都要有值")
|
||||
row.append((value, snap.cell(r, col)))
|
||||
cells[when] = row
|
||||
if bs is ts:
|
||||
env.inputs.use(ts, "data", [params["period"], product, *factors], rows_b + rows_t)
|
||||
else:
|
||||
env.inputs.use(bs, "base", [product, *factors], rows_b)
|
||||
env.inputs.use(ts, "target", [product, *factors], rows_t)
|
||||
outcome, structure = _comparability(env, params, bs, ts, product)
|
||||
if structure is not None:
|
||||
return refuse(env, params, used, "incomparable_sides", f"{structure}，不是同一个量的两期，不拆", criterion=criterion)
|
||||
if any(v <= 0 for row in cells.values() for v, _ in row):
|
||||
return refuse(env, params, used, "non_positive", "乘积或因子里有不大于 0 的值，对数拆解要求全为正", criterion=criterion)
|
||||
for when, row in cells.items():
|
||||
m = row[0][0]
|
||||
prod = math.prod((v for v, _ in row[1:]), start=Fraction(1))
|
||||
rel = abs(m - prod) / abs(m)
|
||||
allowed = max(_FACTOR_REL, sum((_half_ulp_ratio(raw_v, v) for v, raw_v in row[1:]), Fraction(0)))
|
||||
if rel > allowed:
|
||||
raise CallError("not_product", ph.say("这几列相乘不等于", column_label(env, ts, product),
|
||||
f"（{'基期' if when == 'base' else '本期'}差 {ph.pct(rel, signed=False)}），"
|
||||
"不是乘积关系：换成乘起来正好等于它的几列"))
|
||||
M, M2 = cells["base"][0][0], cells["target"][0][0]
|
||||
fb = [float(v) for v, _ in cells["base"][1:]]
|
||||
ft = [float(v) for v, _ in cells["target"][1:]]
|
||||
contrib = lmdi(fb, ft, float(M), float(M2))
|
||||
delta = M2 - M
|
||||
flat = delta == 0 or abs(delta) / abs(M) < env.threshold("flat_pct")
|
||||
shares = [c / float(delta) for c in contrib] if delta != 0 else [None] * len(contrib)
|
||||
main_share = float(env.threshold("main_share"))
|
||||
order = sorted(range(len(factors)), key=lambda i: -abs(contrib[i]))
|
||||
caveats: list[dict[str, str]] = []
|
||||
if outcome is not None and outcome.verdict != "comparable":
|
||||
caveats.append(ph.caveat("comparability", outcome.phrase))
|
||||
total_text = ph.change(delta, M)
|
||||
top = max(range(len(factors)), key=lambda i: shares[i] if shares[i] is not None else -math.inf)
|
||||
if flat:
|
||||
code, subjects = "flat", []
|
||||
main = ph.decompose_flat(label, delta / abs(M), None)
|
||||
elif shares[top] >= main_share: # type: ignore[operator]
|
||||
code, subjects = "driver", [[names[top]]]
|
||||
main = ph.factor_driver(label, total_text, names[top], contrib[top], shares[top])
|
||||
else:
|
||||
code, subjects = "both", []
|
||||
main = ph.factor_both(label, total_text, [(names[i], shares[i]) for i in order])
|
||||
rows = []
|
||||
for i in order:
|
||||
b, t = cells["base"][i + 1][0], cells["target"][i + 1][0]
|
||||
rows.append([names[i], canon(b), canon(t), canon((t - b) / b), canon(contrib[i]),
|
||||
canon(shares[i]) if shares[i] is not None else None])
|
||||
table = {"columns": ["factor", "base", "target", "pct", "contrib", "share"],
|
||||
"labels": {"factor": "因子", "base": "基期", "target": "本期", "pct": "变化率", "contrib": "贡献",
|
||||
"share": "占变化的比重"},
|
||||
"rows": rows}
|
||||
values = {"base_total": canon(M), "target_total": canon(M2), "delta_total": canon(delta),
|
||||
"pct_total": canon(delta / abs(M)), "top_key": names[top],
|
||||
"top_share": canon(shares[top]) if shares[top] is not None else None}
|
||||
verdict = {"code": code, "subjects": subjects, "direction": sign_word(delta) if code != "flat" else None}
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
|
||||
phrase=ph.with_caveats(main, caveats), caveats=caveats)
|
||||
|
||||
|
||||
__all__ = ["HEADLINE_SHAPE", "PARENT_SHAPE", "SAME_QUERY", "SIMPSON_SHARE", "additive", "factor", "lmdi", "ratio", "ratio_effects"]
|
||||
@@ -0,0 +1,372 @@
|
||||
"""差异检验（细案 01 §5）：这两组（或两期）真的不一样，还是样本波动。
|
||||
|
||||
**参数**：query（一份快照）；按组比时 by（分组列，一列）；按期比同一组时 period、base、target（各一行，SQL 里先筛好那一组）。
|
||||
比哪几对：pairs（[[A, B], …]，A、B 是分组列里的值）、reference（对照组：其余各组各和它比）；都不写时两组就比这两组，三组
|
||||
以上每组和「其余各组合计」比（和含它自己的整体比不独立）。可选 label。
|
||||
|
||||
- rate：numerator（成功数，如转化单数）、denominator（试验数，如访客数），每组一行，都要是不小于 0 的整数、成功数不超过
|
||||
试验数。每组 Wilson 区间，两组之差 Newcombe 区间；最小期望频数不小于 min_expected 用合并比例的 z 检验，否则用 Fisher
|
||||
精确检验。
|
||||
- mean：每组一行，mean、sd、n 三列（均值、标准差、样本数），或者 sum、sumsq、n 三列（合计、平方和、样本数：SQLite 没有标准差
|
||||
函数，SUM(x)、SUM(x*x)、COUNT(*) 都写得出）。Welch t 检验，自由度按 Welch–Satterthwaite。
|
||||
- rows：value（逐行的值）+ 分组列（或期次列），快照不超过 1000 行、没截断；算子自己算均值、方差，另做敏感性：去掉离均值
|
||||
最远的那一个值后重算，判定变了加 outlier_sensitive。
|
||||
|
||||
比了几对时按 Holm 校正，表里同时给原始 p 和校正后的 p，附 multiple_comparisons。校正后 p < differ_alpha → differs（direction
|
||||
按 A − B 的正负）；否则 no_evidence。用了 Fisher（比率）或有一组样本数小于 min_n（均值）→ small_sample。判定语不出现「p 值」
|
||||
「显著性」：differs 说「差异站得住」，no_evidence 说「差异可能只是样本波动」，p 值、区间在表和证据面板里。
|
||||
|
||||
subjects 是判定说的那一对 [[A], [B]]（只比一对时就是那一对，比了几对时是校正后最站得住的那一对；几对都不站得住时为空），
|
||||
direction 是 A 相对 B。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis import stats
|
||||
from app.analysis.common import Env, build, column_label, label_for, need, refuse, sign_word
|
||||
from app.analysis.contract import CallError
|
||||
from app.analysis.inputs import Snap, as_list, hashable, same_value
|
||||
from app.analysis.numbers import canon, to_fraction
|
||||
|
||||
TESTS = {"z": "z 检验", "fisher": "Fisher 精确检验", "welch": "Welch t 检验"}
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class Group:
|
||||
key: Any
|
||||
name: str
|
||||
rows: list[int] = field(default_factory=list)
|
||||
x: int = 0 # rate：成功数
|
||||
n: int = 0 # rate：试验数；mean / rows：样本数
|
||||
mean: Fraction | None = None
|
||||
var: Fraction | None = None
|
||||
values: list[Fraction] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Pair:
|
||||
a: Group
|
||||
b: Group
|
||||
test: str | None = None
|
||||
p: float | None = None
|
||||
p_adj: float | None = None
|
||||
diff: Fraction | None = None
|
||||
ci: tuple[float, float] | None = None
|
||||
a_value: Fraction | None = None
|
||||
b_value: Fraction | None = None
|
||||
|
||||
|
||||
def _int(value: Fraction | None, snap: Snap, r: int, column: str) -> int:
|
||||
if value is None or value.denominator != 1 or value < 0:
|
||||
raise CallError("not_count", f"{snap.ref} 第 {r} 行的「{column}」不是不小于 0 的整数：比率比较的分子、分母要是计数")
|
||||
return int(value)
|
||||
|
||||
|
||||
def _pool(name: str, groups: Sequence[Group], mode: str) -> Group:
|
||||
"""「其余各组合计」：比率把成功数、试验数加起来；均值按各组的均值、方差、样本数合并；逐行的值并在一起。"""
|
||||
out = Group(("__rest__",), name, rows=[r for g in groups for r in g.rows])
|
||||
if mode == "rate":
|
||||
out.x, out.n = sum(g.x for g in groups), sum(g.n for g in groups)
|
||||
return out
|
||||
if mode == "rows":
|
||||
out.values = [v for g in groups for v in g.values]
|
||||
return _moments(out)
|
||||
out.n = sum(g.n for g in groups)
|
||||
if out.n == 0:
|
||||
return out
|
||||
out.mean = sum((g.mean * g.n for g in groups if g.mean is not None), Fraction(0)) / out.n
|
||||
if out.n > 1:
|
||||
within = sum(((g.n - 1) * g.var for g in groups if g.var is not None and g.n > 1), Fraction(0))
|
||||
between = sum((g.n * (g.mean - out.mean) ** 2 for g in groups if g.mean is not None), Fraction(0))
|
||||
out.var = (within + between) / (out.n - 1)
|
||||
return out
|
||||
|
||||
|
||||
def _moments(group: Group) -> Group:
|
||||
group.n = len(group.values)
|
||||
if group.n:
|
||||
group.mean = sum(group.values, Fraction(0)) / group.n
|
||||
if group.n > 1:
|
||||
group.var = sum(((v - group.mean) ** 2 for v in group.values), Fraction(0)) / (group.n - 1) # type: ignore[operator]
|
||||
return group
|
||||
|
||||
|
||||
def _test(pair: Pair, mode: str, min_expected: Fraction) -> None:
|
||||
a, b = pair.a, pair.b
|
||||
if mode == "rate":
|
||||
if a.n == 0 or b.n == 0:
|
||||
return
|
||||
pair.a_value, pair.b_value = Fraction(a.x, a.n), Fraction(b.x, b.n)
|
||||
pair.diff = pair.a_value - pair.b_value
|
||||
pair.ci = stats.newcombe(a.x, a.n, b.x, b.n)
|
||||
if stats.min_expected(a.x, a.n, b.x, b.n) >= min_expected:
|
||||
pair.test = "z"
|
||||
pair.p = stats.pooled_z(a.x, a.n, b.x, b.n)[1] # type: ignore[index]
|
||||
else:
|
||||
pair.test = "fisher"
|
||||
pair.p = stats.fisher_exact(a.x, a.n - a.x, b.x, b.n - b.x)
|
||||
return
|
||||
if a.mean is None or b.mean is None:
|
||||
return
|
||||
pair.a_value, pair.b_value = a.mean, b.mean
|
||||
pair.diff = a.mean - b.mean
|
||||
if a.var is None or b.var is None:
|
||||
return
|
||||
test = stats.welch(float(a.mean), float(a.var), a.n, float(b.mean), float(b.var), b.n)
|
||||
if test is None:
|
||||
return
|
||||
pair.test, pair.p = "welch", test.p
|
||||
pair.ci = stats.welch_interval(float(a.mean), float(a.var), a.n, float(b.mean), float(b.var), b.n)
|
||||
|
||||
|
||||
def _verdict(pairs: Sequence[Pair], alpha: Fraction) -> list[Pair]:
|
||||
"""Holm 校正后站得住的那几对（按校正后的 p 从小到大，一样的按先后）。"""
|
||||
adjusted = stats.holm([p.p for p in pairs])
|
||||
for pair, adj in zip(pairs, adjusted):
|
||||
pair.p_adj = adj
|
||||
hits = [p for p in pairs if p.p_adj is not None and to_fraction(p.p_adj) < alpha] # type: ignore[operator]
|
||||
return sorted(hits, key=lambda p: p.p_adj) # type: ignore[arg-type, return-value]
|
||||
|
||||
|
||||
def differ(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
mode = env.mode
|
||||
used = ["differ_alpha", "min_expected"] if mode == "rate" else ["differ_alpha", "min_n"]
|
||||
snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
|
||||
params: dict[str, Any] = {"query": snap.ref}
|
||||
by_period = raw.get("period") is not None
|
||||
if by_period:
|
||||
group_col = snap.column(raw.get("period"), "期次列")
|
||||
params.update(period=group_col, base=need(raw, "base", "基期（期次列里的值）"),
|
||||
target=need(raw, "target", "本期（期次列里的值）"))
|
||||
else:
|
||||
wanted = as_list(raw.get("by"))
|
||||
if len(wanted) != 1:
|
||||
raise CallError("bad_by", "差异检验按一列分组：分组列写一列；比同一组的两期就写期次列、基期、本期")
|
||||
group_col = snap.column(wanted[0], "分组列")
|
||||
params["by"] = [group_col]
|
||||
if mode == "rate":
|
||||
cols = {"numerator": snap.column(need(raw, "numerator", "成功数列"), "成功数列"),
|
||||
"denominator": snap.column(need(raw, "denominator", "试验数列"), "试验数列")}
|
||||
elif mode == "mean":
|
||||
if raw.get("mean") is not None:
|
||||
cols = {"mean": snap.column(raw.get("mean"), "均值列"), "sd": snap.column(need(raw, "sd", "标准差列"), "标准差列"),
|
||||
"n": snap.column(need(raw, "n", "样本数列"), "样本数列")}
|
||||
else:
|
||||
cols = {"sum": snap.column(need(raw, "sum", "合计列（或均值列）"), "合计列"),
|
||||
"sumsq": snap.column(need(raw, "sumsq", "平方和列"), "平方和列"),
|
||||
"n": snap.column(need(raw, "n", "样本数列"), "样本数列")}
|
||||
else:
|
||||
cols = {"value": snap.column(need(raw, "value", "值列"), "值列")}
|
||||
params.update(cols)
|
||||
measure_col = cols.get("numerator") or cols.get("mean") or cols.get("sum") or cols.get("value")
|
||||
if mode == "rate":
|
||||
label = ph.pick_label(raw.get("label"), ph.say(column_label(env, snap, cols["numerator"]), "占",
|
||||
column_label(env, snap, cols["denominator"]), "的比率"))
|
||||
else:
|
||||
label = label_for(env, raw.get("label"), snap, measure_col)
|
||||
if raw.get("label") is not None:
|
||||
params["label"] = ph.clean_label(raw.get("label")) or None
|
||||
dim_label = column_label(env, snap, group_col)
|
||||
criterion = ph.criterion_differ(env.settings, mode) # type: ignore[arg-type]
|
||||
if snap.truncated:
|
||||
env.inputs.use(snap, "data", [group_col, *cols.values()])
|
||||
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，各组不完整",
|
||||
criterion=criterion)
|
||||
groups = _groups(snap, group_col, cols, mode, by_period, params) # type: ignore[arg-type]
|
||||
pairs = _pairs(raw, params, groups, mode, dim_label, by_period, group_col) # type: ignore[arg-type]
|
||||
env.inputs.use(snap, "data", [group_col, *cols.values()],
|
||||
sorted({r for p in pairs for g in (p.a, p.b) for r in g.rows}))
|
||||
return _decide(env, params, used, pairs, mode, label, criterion) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _groups(snap: Snap, group_col: str, cols: Mapping[str, str], mode: str, by_period: bool,
|
||||
params: Mapping[str, Any]) -> dict[Any, Group]:
|
||||
gi = snap.index(group_col)
|
||||
groups: dict[Any, Group] = {}
|
||||
wanted = [params["target"], params["base"]] if by_period else None
|
||||
for r, row in enumerate(snap.rows):
|
||||
value = row[gi] if gi < len(row) else None
|
||||
if wanted is not None:
|
||||
if not any(same_value(w, value) for w in wanted):
|
||||
continue
|
||||
key = hashable(value)
|
||||
group = groups.setdefault(key, Group(key, ph.key_text([value])))
|
||||
if mode != "rows" and group.rows:
|
||||
raise CallError("duplicate_rows", f"{snap.ref} 里「{ph.key_text([value])}」有不止一行：在 SQL 里按 {group_col} "
|
||||
"汇总，每组一行")
|
||||
group.rows.append(r)
|
||||
if mode == "rate":
|
||||
group.x = _int(snap.number(r, cols["numerator"], "成功数列"), snap, r, cols["numerator"])
|
||||
group.n = _int(snap.number(r, cols["denominator"], "试验数列"), snap, r, cols["denominator"])
|
||||
if group.x > group.n:
|
||||
raise CallError("not_count", f"{snap.ref} 第 {r} 行的成功数大于试验数：分子、分母要是同一批里的「成功几个、"
|
||||
"一共几个」")
|
||||
elif mode == "mean":
|
||||
n = _int(snap.number(r, cols["n"], "样本数列"), snap, r, cols["n"])
|
||||
group.n = n
|
||||
if "mean" in cols:
|
||||
group.mean = snap.number(r, cols["mean"], "均值列")
|
||||
sd = snap.number(r, cols["sd"], "标准差列")
|
||||
group.var = sd * sd if sd is not None and n > 1 else None
|
||||
else:
|
||||
total = snap.number(r, cols["sum"], "合计列")
|
||||
sq = snap.number(r, cols["sumsq"], "平方和列")
|
||||
if total is not None and n > 0:
|
||||
group.mean = total / n
|
||||
if sq is not None and n > 1:
|
||||
group.var = max(Fraction(0), (sq - total * total / n) / (n - 1))
|
||||
else:
|
||||
v = snap.number(r, cols["value"], "值列")
|
||||
if v is not None:
|
||||
group.values.append(v)
|
||||
if mode == "rows":
|
||||
for g in groups.values():
|
||||
_moments(g)
|
||||
if by_period:
|
||||
for w, what in ((params["target"], "本期"), (params["base"], "基期")):
|
||||
if not any(same_value(w, g.key) for g in groups.values()):
|
||||
raise CallError("period_not_found", f"期次列「{group_col}」里没有「{w}」：{what}要写 {snap.ref} 里有的期次值")
|
||||
return groups
|
||||
|
||||
|
||||
def _find(groups: Mapping[Any, Group], value: Any, column: str) -> Group:
|
||||
found = next((g for g in groups.values() if same_value(value, g.key)), None)
|
||||
if found is None:
|
||||
shown = "、".join(g.name for g in list(groups.values())[:8])
|
||||
raise CallError("unknown_group", f"「{value}」不是「{column}」里的值：要比的组写 {shown} 里的")
|
||||
return found
|
||||
|
||||
|
||||
def _pairs(raw: Mapping[str, Any], params: dict[str, Any], groups: dict[Any, Group], mode: str, dim_label: str,
|
||||
by_period: bool, group_col: str) -> list[Pair]:
|
||||
if by_period:
|
||||
target = _find(groups, params["target"], group_col)
|
||||
base = _find(groups, params["base"], group_col)
|
||||
return [Pair(target, base)]
|
||||
if raw.get("pairs") is not None:
|
||||
spec = raw.get("pairs")
|
||||
if not isinstance(spec, list) or not spec or not all(isinstance(p, (list, tuple)) and len(p) == 2 for p in spec):
|
||||
raise CallError("bad_pairs", "要比的几对写成 [[A, B], …]，A、B 是分组列里的值")
|
||||
out = []
|
||||
for a, b in spec:
|
||||
ga, gb = _find(groups, a, group_col), _find(groups, b, group_col)
|
||||
if ga is gb:
|
||||
raise CallError("bad_pairs", f"「{ga.name}」不能和自己比")
|
||||
out.append(Pair(ga, gb))
|
||||
params["pairs"] = [[p.a.key, p.b.key] for p in out]
|
||||
return out
|
||||
if len(groups) < 2:
|
||||
raise CallError("too_few_groups", f"「{group_col}」里只有 {len(groups)} 组，至少要两组才比得了")
|
||||
if raw.get("reference") is not None:
|
||||
ref = _find(groups, raw.get("reference"), group_col)
|
||||
params["reference"] = ref.key
|
||||
return [Pair(g, ref) for g in groups.values() if g is not ref]
|
||||
items = list(groups.values())
|
||||
if len(items) == 2:
|
||||
return [Pair(items[0], items[1])]
|
||||
rest_name = ph.say("其余各", dim_label)
|
||||
return [Pair(g, _pool(rest_name, [o for o in items if o is not g], mode)) for g in items]
|
||||
|
||||
|
||||
def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair], mode: str, label: str,
|
||||
criterion: str) -> Any:
|
||||
alpha = env.threshold("differ_alpha")
|
||||
min_expected = env.threshold("min_expected") if mode == "rate" else Fraction(0)
|
||||
for pair in pairs:
|
||||
_test(pair, mode, min_expected)
|
||||
hits = _verdict(pairs, alpha)
|
||||
caveats: list[dict[str, str]] = []
|
||||
if mode == "rows":
|
||||
flipped = _sensitivity(pairs, mode, alpha, min_expected, bool(hits))
|
||||
if flipped is not None:
|
||||
caveats.append(ph.caveat("outlier_sensitive", ph.outlier_text(*flipped)))
|
||||
hits = _verdict(pairs, alpha)
|
||||
if mode == "rate" and any(p.test == "fisher" for p in pairs):
|
||||
caveats.append(ph.caveat("small_sample", ph.small_sample_rate_text(int(env.settings["min_expected"]))))
|
||||
if mode != "rate":
|
||||
min_n = int(env.settings["min_n"])
|
||||
small = sorted((g for p in pairs for g in (p.a, p.b) if g.n < min_n), key=lambda g: g.n)
|
||||
if small:
|
||||
caveats.append(ph.caveat("small_sample", ph.small_sample_mean_text(small[0].name, small[0].n, min_n)))
|
||||
tested = [p for p in pairs if p.p is not None]
|
||||
if len(tested) > 1:
|
||||
caveats.append(ph.caveat("multiple_comparisons", ph.multiple_text(len(tested))))
|
||||
fmt = (lambda v: ph.pct(v, signed=False)) if mode == "rate" else (lambda v: ph.amount(v))
|
||||
if hits:
|
||||
code = "differs"
|
||||
head = hits[0]
|
||||
main = ph.differ_phrase([ph.differ_pair(p.a.name, p.b.name, label, fmt(p.a_value), fmt(p.b_value),
|
||||
p.diff > 0) for p in hits], # type: ignore[operator]
|
||||
len(pairs))
|
||||
subjects = [[head.a.name], [head.b.name]]
|
||||
direction = sign_word(head.diff) # type: ignore[arg-type]
|
||||
else:
|
||||
code = "no_evidence"
|
||||
head = pairs[0] if len(pairs) == 1 else None
|
||||
sizes = (ph.amount(head.a.n), ph.amount(head.b.n)) if head is not None else None
|
||||
main = ph.no_evidence_phrase(head.a.name if head else None, head.b.name if head else None, label, sizes,
|
||||
len(pairs))
|
||||
subjects = [[head.a.name], [head.b.name]] if head is not None else []
|
||||
direction = None
|
||||
value_name = "比率" if mode == "rate" else "均值"
|
||||
rows = []
|
||||
for p in pairs:
|
||||
flags = []
|
||||
if p.p is None:
|
||||
flags.append("样本不够，算不了")
|
||||
if p.test == "fisher":
|
||||
flags.append("样本少，用了精确检验")
|
||||
rows.append([p.a.name, p.b.name, canon(p.a_value), canon(p.b_value), canon(p.diff),
|
||||
canon(p.ci[0]) if p.ci else None, canon(p.ci[1]) if p.ci else None, str(p.a.n), str(p.b.n),
|
||||
TESTS.get(p.test or "") or None, canon(p.p), canon(p.p_adj), "、".join(flags) or None])
|
||||
table = {"columns": ["a", "b", "a_value", "b_value", "diff", "ci_lo", "ci_hi", "a_n", "b_n", "test", "p", "p_adj",
|
||||
"flag"],
|
||||
"labels": {"a": "组", "b": "对照组", "a_value": f"组的{value_name}", "b_value": f"对照组的{value_name}",
|
||||
"diff": "差", "ci_lo": "差的区间下限", "ci_hi": "差的区间上限", "a_n": "组的样本量",
|
||||
"b_n": "对照组的样本量", "test": "检验", "p": "p 值", "p_adj": "校正后的 p 值", "flag": "说明"},
|
||||
"rows": rows}
|
||||
shown = head if head is not None else pairs[0]
|
||||
values = {"a": canon(shown.a_value), "b": canon(shown.b_value), "diff": canon(shown.diff),
|
||||
"ci_lo": canon(shown.ci[0]) if shown.ci else None, "ci_hi": canon(shown.ci[1]) if shown.ci else None,
|
||||
"p": canon(shown.p), "p_adj": canon(shown.p_adj), "n_a": str(shown.a.n), "n_b": str(shown.b.n),
|
||||
"test": shown.test, "pairs": str(len(pairs)), "differs": str(len(hits))}
|
||||
verdict = {"code": code, "subjects": subjects, "direction": direction}
|
||||
phrase = ph.with_caveats(main, caveats, skip=("multiple_comparisons",) if len(pairs) > 1 else ())
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, phrase=phrase,
|
||||
caveats=caveats)
|
||||
|
||||
|
||||
def _sensitivity(pairs: list[Pair], mode: str, alpha: Fraction, min_expected: Fraction,
|
||||
differs: bool) -> tuple[bool, bool] | None:
|
||||
"""rows 模式的敏感性：去掉比较的各组里离均值最远的那一个值重算，判定变了交回 (去掉的是不是偏大的那一笔, 现在站不站得住)。"""
|
||||
everything = [(g, i, v) for g in dict.fromkeys(g for p in pairs for g in (p.a, p.b)) for i, v in enumerate(g.values)]
|
||||
if len(everything) < 3:
|
||||
return None
|
||||
mean = sum((v for _, _, v in everything), Fraction(0)) / len(everything)
|
||||
group, index, value = max(everything, key=lambda t: abs(t[2] - mean))
|
||||
trimmed = []
|
||||
for p in pairs:
|
||||
def cut(g: Group) -> Group:
|
||||
if g is not group and not (g.key == ("__rest__",) and any(v is value for v in g.values)):
|
||||
return g
|
||||
values = list(g.values)
|
||||
for k, v in enumerate(values):
|
||||
if v is value:
|
||||
del values[k]
|
||||
break
|
||||
return _moments(Group(g.key, g.name, rows=g.rows, values=values))
|
||||
trimmed.append(Pair(cut(p.a), cut(p.b)))
|
||||
for p in trimmed:
|
||||
_test(p, mode, min_expected)
|
||||
now = bool(_verdict(trimmed, alpha))
|
||||
_verdict(pairs, alpha)
|
||||
return (value > mean, now) if now != differs else None
|
||||
|
||||
|
||||
__all__ = ["TESTS", "differ"]
|
||||
@@ -198,6 +198,8 @@ class Inputs:
|
||||
out["artifact"] = snap.artifact
|
||||
if snap.dialect:
|
||||
out["dialect"] = snap.dialect
|
||||
if snap.masked:
|
||||
out["masked"] = sorted(snap.masked) # 只记列名（复算按它认遮罩列，快照工件里不一定都有）
|
||||
out.update({k: v for k, v in entry.items() if k != "ref"})
|
||||
if out not in self.records:
|
||||
self.records.append(out) # type: ignore[arg-type]
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
"""放回历史看（细案 01 §3、§4）：波动基线（这个变化大不大、算不算异常）和趋势（是持续在变，还是一次波动）。
|
||||
|
||||
**参数**：query（一份快照：一条序列；要看某一段就在 SQL 里筛好）、period（期次列）、measure（度量列）；可选 grain（day /
|
||||
week / month / quarter，缺省按期次值推断，见 analysis/periods）、label。波动基线另有 target（本期，缺省最后一期）、basis
|
||||
（change 缺省：和比较期比的变化率；level：值本身，给存量、比率用）、lag（和往前第几期比，缺省日数据 7、其余 1；同比写 52 /
|
||||
12 / 4，历史要相应够长）。
|
||||
|
||||
**前提**（不满足就算不了，原因照实写）：期次认得出（period_unreadable）、本期完整（同可比性检查的整期：没过完、数据没到、
|
||||
SQL 截掉一部分 → partial_target；表的日期范围查不到时只按运行当天判，附一句核对不了）、本期和比较期之间不缺期（target_gap）、
|
||||
比较期不为 0（zero_base）、历史够长（history_short）；趋势要最后几期连着（中间缺期 series_gap）。度量格为空的那一期跳过。
|
||||
|
||||
**波动基线**：历史的中位数 m、中位绝对偏差 MAD，σ̂ = 1.4826·MAD，稳健 z = (x − m)/σ̂；常规波动范围 m ± k_warn·σ̂；全程
|
||||
精确的有理数。σ̂ = 0（历史每期都一样）时本期一样判常规、不一样判明显超出。本期的值（不论 basis）是历史各期里最高 / 最低的
|
||||
加 record_high / record_low：「历史新高」说的是值本身。为什么不用均值和标准差：促销周这类极端值会把它们拉宽。
|
||||
|
||||
**趋势**：最后 n 期（期次连着的那一段，trend_min_points ≤ n ≤ max_history + 1）的 Theil–Sen 斜率、Mann–Kendall 双侧 p，
|
||||
相对斜率 = 斜率 / |序列中位数|。p < trend_alpha 时按斜率的正负判上升、下降。末尾连续同向的期数从最后一期往前数（变化为 0
|
||||
断开）。已知局限：Mann–Kendall 假设各期独立，业务序列常有自相关，p 会偏小；所以缺省 n 至少 6、α 取 0.05，判定语只说
|
||||
「呈上升趋势」，不说「显著上升」。日数据附一句星期效应。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
from app.analysis import comparable as cmp
|
||||
from app.analysis import periods as pd
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis import stats
|
||||
from app.analysis.common import Env, build, column_label, label_for, need, refuse, sign_word, today_param
|
||||
from app.analysis.contract import CallError
|
||||
from app.analysis.inputs import Snap, same_value
|
||||
from app.analysis.numbers import canon, to_fraction
|
||||
|
||||
#: 稳健标准差 = 1.4826 × MAD（正态时等于标准差）
|
||||
MAD_SCALE = Fraction(14826, 10000)
|
||||
#: lag 的上限（周数据同比是 52）
|
||||
_LAG_MAX = 260
|
||||
|
||||
|
||||
@dataclass
|
||||
class Series:
|
||||
snap: Snap
|
||||
period: str
|
||||
measure: str
|
||||
grain: str
|
||||
label: str
|
||||
points: list[tuple[pd.Period, Fraction]] # 按期次排好、度量不为空的
|
||||
nulls: int
|
||||
params: dict[str, Any]
|
||||
|
||||
|
||||
def _load(raw: Mapping[str, Any], env: Env, used: list[str]) -> Series | Any:
|
||||
"""读出一条序列；算不了时直接交回 refused 的结论。"""
|
||||
snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
|
||||
period = snap.column(need(raw, "period", "期次列"), "期次列")
|
||||
measure = snap.column(need(raw, "measure", "度量列"), "度量列")
|
||||
grain = raw.get("grain")
|
||||
if grain is not None and grain not in (*pd.GRAINS, "seq"):
|
||||
raise CallError("bad_grain", "粒度只能写 day、week、month、quarter，或者不写（按期次值推断）")
|
||||
params: dict[str, Any] = {"query": snap.ref, "period": period, "measure": measure}
|
||||
label = label_for(env, raw.get("label"), snap, measure)
|
||||
if raw.get("label") is not None:
|
||||
params["label"] = ph.clean_label(raw.get("label")) or None
|
||||
criterion = _criterion(env, None, "期")
|
||||
env.inputs.use(snap, "data", [period, measure])
|
||||
if snap.truncated:
|
||||
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，序列不完整",
|
||||
criterion=criterion)
|
||||
i_period = snap.index(period)
|
||||
values = [row[i_period] if i_period < len(row) else None for row in snap.rows]
|
||||
parsed = pd.parse(values, grain)
|
||||
if parsed is None:
|
||||
sample = "、".join(dict.fromkeys(ph.key_text([v]) for v in values[:3]))
|
||||
return refuse(env, params, used, "period_unreadable",
|
||||
f"期次列「{period}」的值认不出是哪一期（{sample}）：要日期、YYYY-MM、YYYY-Www、YYYY-Qn 或整数序号",
|
||||
criterion=criterion)
|
||||
items, grain = parsed
|
||||
params["grain"] = grain
|
||||
seen: set[int] = set()
|
||||
points: list[tuple[pd.Period, Fraction]] = []
|
||||
nulls = 0
|
||||
for r, item in enumerate(items):
|
||||
if item.ordinal in seen:
|
||||
raise CallError("duplicate_rows", ph.say(f"{snap.ref} 里同一期有不止一行：在 SQL 里按 ",
|
||||
column_label(env, snap, period), " 汇总，每期一行"))
|
||||
seen.add(item.ordinal)
|
||||
value = snap.number(r, measure, "度量列")
|
||||
if value is None:
|
||||
nulls += 1
|
||||
continue
|
||||
points.append((item, value))
|
||||
points.sort(key=lambda p: p[0].ordinal)
|
||||
return Series(snap, period, measure, grain, label, points, nulls, params)
|
||||
|
||||
|
||||
def _criterion(env: Env, n: int | None, unit: str) -> str:
|
||||
if env.op == "baseline":
|
||||
return ph.criterion_baseline(env.settings, n, unit)
|
||||
return ph.criterion_trend(env.settings, n, unit)
|
||||
|
||||
|
||||
def _completeness(env: Env, series: Series, item: pd.Period) -> tuple[str, str | None]:
|
||||
"""本期齐不齐（同可比性检查的整期）：(pass / short / unknown, 不齐的那一句)。"""
|
||||
series.params["today"] = today_param(env)
|
||||
if item.start is None:
|
||||
return "unknown", None
|
||||
span = cmp.period_span("本期", series.snap, series.period, item, series.grain, series.measure, env.facts, env.book)
|
||||
return cmp.span_coverage(span, env.today, env.book) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _shown_period(item: pd.Period) -> Any:
|
||||
return item.raw
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 波动基线
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def baseline(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["k_warn", "k_alert", "min_history", "max_history"]
|
||||
loaded = _load(raw, env, used)
|
||||
if not isinstance(loaded, Series):
|
||||
return loaded
|
||||
series, params = loaded, loaded.params
|
||||
basis = raw.get("basis") or "change"
|
||||
if basis not in ("change", "level"):
|
||||
raise CallError("bad_basis", "比法只能写 change（和比较期比的变化率）或 level（值本身）")
|
||||
lag = raw.get("lag", pd.DEFAULT_LAG[series.grain])
|
||||
if isinstance(lag, bool) or not isinstance(lag, int) or not 1 <= lag <= _LAG_MAX:
|
||||
raise CallError("bad_lag", f"往前比几期要写 1 到 {_LAG_MAX} 的整数（同比：周数据 52、月数据 12、季度 4）")
|
||||
params.update(basis=basis, lag=lag)
|
||||
unit, per = pd.COUNT_WORD[series.grain], pd.PER_WORD[series.grain]
|
||||
criterion = _criterion(env, None, unit)
|
||||
if not series.points:
|
||||
return refuse(env, params, used, "history_short", f"只有 0 期历史，至少要 {int(env.settings['min_history'])} 期",
|
||||
criterion=criterion)
|
||||
if raw.get("target") is not None:
|
||||
found = next((p for p in series.points if same_value(raw["target"], p[0].raw)), None)
|
||||
if found is None:
|
||||
raise CallError("period_not_found", f"期次列「{series.period}」里没有「{raw['target']}」（或那一期为空）："
|
||||
"本期要写序列里有值的一期，不写就是最后一期")
|
||||
else:
|
||||
found = series.points[-1]
|
||||
target, x_level = found
|
||||
params["target"] = target.raw
|
||||
caveats: list[dict[str, str]] = []
|
||||
status, why = _completeness(env, series, target)
|
||||
if status == "short":
|
||||
return refuse(env, params, used, "partial_target", f"{why}，和完整的{per}比没有意义", criterion=criterion)
|
||||
if status == "unknown" and target.start is not None:
|
||||
caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE))
|
||||
by_ordinal = {p.ordinal: v for p, v in series.points}
|
||||
before = [(p, v) for p, v in series.points if p.ordinal < target.ordinal]
|
||||
if basis == "change":
|
||||
prev = by_ordinal.get(target.ordinal - lag)
|
||||
if prev is None:
|
||||
return refuse(env, params, used, "target_gap",
|
||||
f"本期和它的比较期（往前第 {lag} {'期' if series.grain == 'seq' else unit}）之间缺期，没有比较期的值",
|
||||
criterion=criterion)
|
||||
if prev == 0:
|
||||
return refuse(env, params, used, "zero_base", ph.say("比较期的", series.label, "为 0，算不了变化率"),
|
||||
criterion=criterion)
|
||||
x = (x_level - prev) / abs(prev)
|
||||
history = [(p, (v - by_ordinal[p.ordinal - lag]) / abs(by_ordinal[p.ordinal - lag]), v) for p, v in before
|
||||
if by_ordinal.get(p.ordinal - lag) not in (None, 0)]
|
||||
else:
|
||||
x = x_level
|
||||
history = [(p, v, v) for p, v in before]
|
||||
history = history[-int(env.settings["max_history"]):]
|
||||
n = len(history)
|
||||
criterion = _criterion(env, n, unit)
|
||||
if n < int(env.settings["min_history"]):
|
||||
return refuse(env, params, used, "history_short", f"只有 {n} 期历史，至少要 {int(env.settings['min_history'])} 期",
|
||||
criterion=criterion)
|
||||
hv = [h for _, h, _ in history]
|
||||
m = stats.median(hv)
|
||||
sigma = MAD_SCALE * stats.mad(hv, m)
|
||||
k_warn, k_alert = env.threshold("k_warn"), env.threshold("k_alert")
|
||||
if sigma == 0:
|
||||
z = None
|
||||
code = "within_band" if x == m else "far_outside_band"
|
||||
else:
|
||||
z = (x - m) / sigma
|
||||
code = "within_band" if abs(z) < k_warn else "outside_band" if abs(z) < k_alert else "far_outside_band"
|
||||
lo, hi = m - k_warn * sigma, m + k_warn * sigma
|
||||
levels = [lv for _, _, lv in history]
|
||||
if x_level > max(levels):
|
||||
caveats.append(ph.caveat("record_high", ph.record_text(True, series.label, n, unit)))
|
||||
elif x_level < min(levels):
|
||||
caveats.append(ph.caveat("record_low", ph.record_text(False, series.label, n, unit)))
|
||||
if series.nulls:
|
||||
caveats.append(ph.caveat("null_values", ph.null_periods_text(series.nulls, series.label)))
|
||||
current = ph.current_text(x, basis=basis, lag=lag, grain=series.grain)
|
||||
if sigma == 0:
|
||||
shown = ph.pct(m) if basis == "change" else ph.amount(m)
|
||||
main = ph.baseline_flat_phrase(code, series.label, current, n, unit, per, shown)
|
||||
else:
|
||||
main = ph.baseline_phrase(code, series.label, current, n, unit, ph.band_text(lo, hi, basis=basis))
|
||||
rows = [[p.raw, canon(lv), canon(h) if basis == "change" else None, "历史"] for p, h, lv in history]
|
||||
rows.append([target.raw, canon(x_level), canon(x) if basis == "change" else None, "本期"])
|
||||
table = {"columns": [series.period, "value", "change", "role"],
|
||||
"labels": {series.period: column_label(env, series.snap, series.period), "value": series.label,
|
||||
"change": "变化率", "role": "说明"},
|
||||
"rows": rows}
|
||||
values = {"x": canon(x), "value": canon(x_level), "median": canon(m), "band_lo": canon(lo), "band_hi": canon(hi),
|
||||
"z": canon(z) if z is not None else None, "rank": str(stats.rank_desc(x, hv)), "n_history": str(n),
|
||||
"hist_min": canon(min(hv)), "hist_max": canon(max(hv))}
|
||||
verdict = {"code": code, "subjects": [], "direction": sign_word(x - m)}
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
|
||||
phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 趋势
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_length(values: list[Fraction]) -> tuple[int, str | None]:
|
||||
"""末尾连续同向：从最后一期往前数，相邻变化同号的有几个（变化为 0 断开）。"""
|
||||
changes = [b - a for a, b in zip(values, values[1:])]
|
||||
if not changes or changes[-1] == 0:
|
||||
return 0, None
|
||||
sign = changes[-1] > 0
|
||||
count = 0
|
||||
for c in reversed(changes):
|
||||
if c == 0 or (c > 0) != sign:
|
||||
break
|
||||
count += 1
|
||||
return count, "up" if sign else "down"
|
||||
|
||||
|
||||
def trend(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["trend_min_points", "trend_alpha", "max_history"]
|
||||
loaded = _load(raw, env, used)
|
||||
if not isinstance(loaded, Series):
|
||||
return loaded
|
||||
series, params = loaded, loaded.params
|
||||
unit, per = pd.COUNT_WORD[series.grain], pd.PER_WORD[series.grain]
|
||||
minimum = int(env.settings["trend_min_points"])
|
||||
criterion = _criterion(env, None, unit)
|
||||
if not series.points:
|
||||
return refuse(env, params, used, "history_short", f"只有 0 期，至少要 {minimum} 期", criterion=criterion)
|
||||
last = series.points[-1][0]
|
||||
caveats: list[dict[str, str]] = []
|
||||
status, why = _completeness(env, series, last)
|
||||
if status == "short":
|
||||
return refuse(env, params, used, "partial_target", f"{why}，和完整的{per}比没有意义", criterion=criterion)
|
||||
if status == "unknown" and last.start is not None:
|
||||
caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE))
|
||||
cap = int(env.settings["max_history"]) + 1
|
||||
tail = [series.points[-1]]
|
||||
for point in reversed(series.points[:-1]):
|
||||
if len(tail) >= cap or point[0].ordinal != tail[-1][0].ordinal - 1:
|
||||
break
|
||||
tail.append(point)
|
||||
tail.reverse()
|
||||
n = len(tail)
|
||||
criterion = _criterion(env, n, unit)
|
||||
if n < minimum:
|
||||
gap = len(series.points) > n and len(tail) < cap
|
||||
if gap:
|
||||
return refuse(env, params, used, "series_gap",
|
||||
f"最后连着的只有 {n} 期，再往前缺期（至少要连着的 {minimum} 期）", criterion=criterion)
|
||||
return refuse(env, params, used, "history_short", f"只有 {n} 期，至少要 {minimum} 期", criterion=criterion)
|
||||
ys = [v for _, v in tail]
|
||||
xs = [Fraction(i) for i in range(n)]
|
||||
slope = stats.theil_sen(xs, ys)
|
||||
med = stats.median(ys)
|
||||
rel = slope / abs(med) if med != 0 else None
|
||||
mk = stats.mann_kendall(ys)
|
||||
alpha = env.threshold("trend_alpha")
|
||||
p = mk.p if isinstance(mk.p, Fraction) else to_fraction(mk.p)
|
||||
if p is not None and p < alpha and slope > 0:
|
||||
code = "rising"
|
||||
elif p is not None and p < alpha and slope < 0:
|
||||
code = "falling"
|
||||
else:
|
||||
code = "no_trend"
|
||||
run, direction = run_length(ys)
|
||||
if series.grain == "day":
|
||||
caveats.append(ph.caveat("weekday_effect", ph.WEEKDAY_EFFECT))
|
||||
if series.nulls:
|
||||
caveats.append(ph.caveat("null_values", ph.null_periods_text(series.nulls, series.label)))
|
||||
slope_text = ph.pct(rel) if rel is not None else ph.amount(slope, signed=True)
|
||||
main = ph.trend_phrase(code, series.label, n, unit, per, slope_text,
|
||||
(run, direction) if run >= 3 and direction else None) # type: ignore[arg-type]
|
||||
rows = [[p_.raw, canon(v), canon(v - tail[i - 1][1]) if i else None] for i, (p_, v) in enumerate(tail)]
|
||||
table = {"columns": [series.period, "value", "change"],
|
||||
"labels": {series.period: column_label(env, series.snap, series.period), "value": series.label,
|
||||
"change": "比上一期"},
|
||||
"rows": rows}
|
||||
values = {"slope": canon(slope), "rel_slope": canon(rel) if rel is not None else None, "p": canon(mk.p),
|
||||
"s": str(mk.s), "n": str(n), "run_length": str(run), "run_direction": direction}
|
||||
verdict = {"code": code, "subjects": [], "direction": "up" if code == "rising" else "down" if code == "falling"
|
||||
else None}
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
|
||||
phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats)
|
||||
|
||||
|
||||
__all__ = ["MAD_SCALE", "baseline", "run_length", "trend"]
|
||||
@@ -0,0 +1,127 @@
|
||||
"""分析算子测试用的样例：虚构零售库 shop（地区华东、华南、华北、西南，渠道线上、线下，销售额、订单数、退款），表名、列名、
|
||||
数字都是编的。analyze 跑一次 analysis.run，顺手用 recompute 复算一遍、核对逐字段相同（contract.same），所以每个用例都同时
|
||||
钉住「复算和运行逐字段相同」。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from app import analysis
|
||||
from app.analysis.contract import CallError, effective_settings, same
|
||||
from app.core.artifact_store import canonical_json, content_hash
|
||||
|
||||
#: 测试里的「运行当天」
|
||||
TODAY = date(2026, 10, 21)
|
||||
|
||||
|
||||
def snap(ref: str, sql: str, columns: Sequence[str], rows: Sequence[Sequence[Any]], *,
|
||||
types: Mapping[str, str] | None = None, source: str = "shop", truncated: bool = False,
|
||||
masked: Sequence[str] = (), **extra: Any) -> dict[str, Any]:
|
||||
"""一份输入快照（InputSnapshot 的形状）。artifact 按内容算，复算按它取回。"""
|
||||
body = {"source": source, "sql": sql, "columns": list(columns), "rows": [list(r) for r in rows],
|
||||
"truncated": truncated, **({"column_types": dict(types)} if types else {}), **extra}
|
||||
return {"ref": ref, "artifact": content_hash(canonical_json(body)), "node_id": "q1", "masked": list(masked), **body}
|
||||
|
||||
|
||||
def item(value: Any, status: str = "confirmed", source: str = "human") -> dict[str, Any]:
|
||||
return {"value": value, "source": source, "status": status}
|
||||
|
||||
|
||||
def notes(*, columns: Mapping[str, Mapping[str, Any]] | None = None, business_date: str | None = None) -> dict[str, Any]:
|
||||
"""一张表的目录 notes：columns 是 {列: {label / measure: 值或 item(...)}}。"""
|
||||
out: dict[str, Any] = {}
|
||||
if columns:
|
||||
out["columns"] = {c: {k: v if isinstance(v, Mapping) else item(v) for k, v in spec.items()}
|
||||
for c, spec in columns.items()}
|
||||
if business_date:
|
||||
out["business_date"] = item({"column": business_date, "rule": "下单日", "timezone": "Asia/Shanghai"})
|
||||
return out
|
||||
|
||||
|
||||
#: orders 表的目录：地区、渠道、实收金额（流量）、退款单数
|
||||
ORDERS = notes(columns={"region": {"label": "地区"}, "channel": {"label": "渠道"},
|
||||
"amount": {"label": "销售额", "measure": "flow"},
|
||||
"refund_flag": {"label": "退款单数", "measure": "flow"},
|
||||
"order_date": {"label": "下单日"}},
|
||||
business_date="order_date")
|
||||
CATALOGS = {"shop": {"orders": ORDERS}}
|
||||
|
||||
|
||||
async def analyze(op: str, mode: str | None = None, *, snaps: Sequence[Mapping[str, Any]], today: date = TODAY,
|
||||
catalogs: Mapping[str, Any] | None = None, probe: Any = None, facts: Any = None,
|
||||
parents: Mapping[str, Mapping[str, Any]] | None = None, settings: Mapping[str, Any] | None = None,
|
||||
hypothesis: str | None = None, check: bool = True, **params: Any) -> dict[str, Any]:
|
||||
"""跑一次 analysis.run；check 时再复算一遍，核对和封存的逐字段相同（hypothesis 除外）。"""
|
||||
by_ref = {s["ref"]: s for s in snaps}
|
||||
request: dict[str, Any] = {"op": op, "mode": mode, "params": params}
|
||||
if hypothesis is not None:
|
||||
request["hypothesis"] = hypothesis
|
||||
context: dict[str, Any] = {"today": today, "catalogs": CATALOGS if catalogs is None else catalogs}
|
||||
if probe is not None:
|
||||
context["probe"] = probe
|
||||
if facts is not None:
|
||||
context["facts"] = facts
|
||||
result = await analysis.run(request, by_ref.get, settings=effective_settings(settings), context=context,
|
||||
parents=(parents or {}).get)
|
||||
if check:
|
||||
again = recomputed(result, snaps, parents)
|
||||
assert same(result, again), _diff(result, again)
|
||||
assert again["hypothesis"] == result["hypothesis"]
|
||||
return dict(result)
|
||||
|
||||
|
||||
def recomputed(result: Mapping[str, Any], snaps: Sequence[Mapping[str, Any]],
|
||||
parents: Mapping[str, Mapping[str, Any]] | None = None) -> dict[str, Any]:
|
||||
"""按结论里记下的复算：快照按 artifact 取回（只交快照内容本身，不交入口加的出处），上一层结论按内容哈希取回。"""
|
||||
by_artifact = {s["artifact"]: {k: v for k, v in s.items() if k not in ("ref", "artifact", "node_id", "masked")}
|
||||
for s in snaps}
|
||||
by_hash = {content_hash(canonical_json(r)): r for r in (parents or {}).values()}
|
||||
return dict(analysis.recompute(result, by_artifact.get, load_result=by_hash.get)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _diff(a: Mapping[str, Any], b: Mapping[str, Any]) -> str:
|
||||
keys = sorted(k for k in set(a) | set(b) if a.get(k) != b.get(k))
|
||||
return "；".join(f"{k}: {json.dumps(a.get(k), ensure_ascii=False)} ≠ {json.dumps(b.get(k), ensure_ascii=False)}"
|
||||
for k in keys)
|
||||
|
||||
|
||||
async def call_error(op: str, mode: str | None = None, **kwargs: Any) -> CallError:
|
||||
"""跑一次，期望是调用错误。"""
|
||||
try:
|
||||
await analyze(op, mode, check=False, **kwargs)
|
||||
except CallError as e:
|
||||
return e
|
||||
raise AssertionError("应当是调用错误")
|
||||
|
||||
|
||||
def fixed_prober(ranges: Mapping[tuple[str, str], tuple[str | None, str | None]] | None = None, *,
|
||||
bounds: Mapping[str, str] | None = None, at: str = "2026-10-21T09:00:00") -> Any:
|
||||
"""测试用的探查器：ranges 是 {(表, 列): (最早, 最晚)}，bounds 是 {界的 SQL: 日期}；没写的查不到。"""
|
||||
|
||||
class Prober:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, ...]] = []
|
||||
|
||||
async def __call__(self, source: str, table: str, column: str) -> dict[str, Any] | None:
|
||||
self.calls.append((source, table, column))
|
||||
found = (ranges or {}).get((table, column))
|
||||
if found is None:
|
||||
return None
|
||||
return {"source": source, "table": table, "column": column, "min": found[0], "max": found[1],
|
||||
"basis": "live", "at": at}
|
||||
|
||||
async def bounds(self, source: str, expr: str) -> dict[str, Any] | None:
|
||||
self.calls.append((source, expr))
|
||||
value = (bounds or {}).get(expr)
|
||||
return None if value is None else {"source": source, "expr": expr, "value": value, "basis": "live", "at": at}
|
||||
|
||||
def dialect(self, source: str) -> str | None:
|
||||
return "sqlite"
|
||||
|
||||
return Prober()
|
||||
|
||||
|
||||
__all__ = ["CATALOGS", "ORDERS", "TODAY", "analyze", "call_error", "fixed_prober", "item", "notes", "recomputed",
|
||||
"snap"]
|
||||
@@ -0,0 +1,182 @@
|
||||
"""可比性检查（app/analysis/comparable.py，细案 01 §6）：只差日期 / 差了别的筛选 / 结构不同、窗口不等（流量不可比、比率只提醒）、
|
||||
本期不完整、周末天数不同、探查失败退回剖析事实、都没有就核对不了。每个用例都顺带核对复算和运行逐字段相同（复算用记下的
|
||||
probes，不现查）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from analysis_samples import analyze, fixed_prober, snap
|
||||
from app.analysis import comparable_reasons
|
||||
|
||||
WINDOW = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= '{a}' AND order_date < '{b}'"
|
||||
RANGE = {("orders", "order_date"): ("2025-01-01", "2026-10-20")}
|
||||
|
||||
|
||||
def period(ref, a, b, sql=WINDOW, **kw):
|
||||
return snap(ref, sql.format(a=a, b=b), ["gmv"], [[100]], types={"gmv": "number"}, **kw)
|
||||
|
||||
|
||||
async def compare(base, target, *, probe=None, **params):
|
||||
params = {"base_query": base["ref"], "target_query": target["ref"], **params}
|
||||
return await analyze("comparable", snaps=[base, target], probe=probe, **params)
|
||||
|
||||
|
||||
def checks(result):
|
||||
return {r[0]: r[1] for r in result["table"]["rows"]}
|
||||
|
||||
|
||||
async def test_two_weeks_that_differ_only_in_dates_are_comparable():
|
||||
result = await compare(period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19"),
|
||||
probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["method"] == "comparable@1" and result["mode"] is None
|
||||
assert result["verdict"] == {"code": "comparable", "subjects": [], "direction": None}
|
||||
assert result["phrase"] == "两期可比"
|
||||
assert checks(result) == {"同口径": "通过", "窗口长度": "通过", "整期": "通过", "星期构成": "通过", "数据问题": "通过"}
|
||||
assert result["values"]["base_days"] == "7" and result["values"]["target_start"] == "2026-10-12"
|
||||
assert result["values"]["window"] == "pass" and result["settings"] == {}
|
||||
assert result["params"]["today"] == "2026-10-21" and result["params"]["measure"] == "gmv"
|
||||
assert [p["column"] for p in result["probes"]] == ["order_date"]
|
||||
assert result["table"]["columns"] == ["check", "status", "base", "target", "note"]
|
||||
|
||||
|
||||
async def test_other_literals_differing_mean_two_segments_not_two_periods():
|
||||
sql = WINDOW + " AND region = '{r}'"
|
||||
base = snap("Q1", sql.format(a="2026-10-05", b="2026-10-12", r="华南"), ["gmv"], [[100]])
|
||||
target = snap("Q2", sql.format(a="2026-10-12", b="2026-10-19", r="华东"), ["gmv"], [[100]])
|
||||
result = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["verdict"]["code"] == "incomparable"
|
||||
assert result["phrase"] == "两期不可比：两边除了日期，筛选的还不一样"
|
||||
assert comparable_reasons(result) == ["两边除了日期，筛选的还不一样"]
|
||||
|
||||
|
||||
async def test_a_different_structure_says_where_it_differs():
|
||||
base = period("Q1", "2026-10-05", "2026-10-12")
|
||||
target = period("Q2", "2026-10-12", "2026-10-19", sql=WINDOW + " AND status = 'paid'")
|
||||
result = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["verdict"]["code"] == "incomparable"
|
||||
assert result["phrase"] == "两期不可比：两份查询的口径结构不同（差在筛选条件）"
|
||||
other = await compare(base, period("Q2", "2026-10-12", "2026-10-19", source="crm"), measure="gmv")
|
||||
assert "两份查询来自不同的数据源" in other["phrase"]
|
||||
|
||||
|
||||
async def test_unequal_windows_fail_for_flows_and_only_warn_otherwise():
|
||||
base, target = period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-15")
|
||||
flow = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv") # SUM(amount)：目录标的流量
|
||||
assert flow["verdict"]["code"] == "incomparable" and checks(flow)["窗口长度"] == "不通过"
|
||||
assert "两段的天数不同：7 天对 3 天" in flow["phrase"]
|
||||
avg_sql = "SELECT AVG(amount) AS aov FROM orders WHERE order_date >= '{a}' AND order_date < '{b}'"
|
||||
ratio = await compare(period("Q1", "2026-10-05", "2026-10-12", sql=avg_sql),
|
||||
period("Q2", "2026-10-12", "2026-10-15", sql=avg_sql), probe=fixed_prober(RANGE),
|
||||
measure="gmv")
|
||||
assert ratio["verdict"]["code"] == "conditional" and checks(ratio)["窗口长度"] == "提醒"
|
||||
unlabeled = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv", catalogs={})
|
||||
assert unlabeled["verdict"]["code"] == "conditional"
|
||||
assert any(c["code"] == "measure_unlabeled" for c in unlabeled["caveats"])
|
||||
|
||||
|
||||
async def test_whole_months_count_as_the_same_length():
|
||||
result = await compare(period("Q1", "2026-08-01", "2026-09-01"), period("Q2", "2026-09-01", "2026-10-01"),
|
||||
probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["verdict"]["code"] == "comparable" and result["values"]["base_days"] == "31"
|
||||
|
||||
|
||||
async def test_an_unfinished_period_is_not_a_whole_period():
|
||||
result = await compare(period("Q1", "2026-10-12", "2026-10-19"), period("Q2", "2026-10-19", "2026-10-26"),
|
||||
probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["verdict"]["code"] == "incomparable"
|
||||
assert result["phrase"] == "两期不可比：本期数据只到 2026-10-20，还差 5 天"
|
||||
late = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-16")})
|
||||
behind = await compare(period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19"),
|
||||
probe=late, measure="gmv")
|
||||
assert behind["phrase"] == "两期不可比：本期数据只到 2026-10-16，还差 2 天"
|
||||
|
||||
|
||||
async def test_weekend_composition_of_short_windows():
|
||||
# 周一到周三 对 周五到周日
|
||||
result = await compare(period("Q1", "2026-10-05", "2026-10-08"), period("Q2", "2026-10-16", "2026-10-19"),
|
||||
probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["verdict"]["code"] == "conditional"
|
||||
assert result["phrase"] == "两期有条件可比：两段的周末天数不同：0 天对 2 天"
|
||||
|
||||
|
||||
async def test_a_failed_probe_falls_back_to_profile_facts_then_says_it_cannot_check():
|
||||
facts = {"shop": {"orders": {"at": "2026-10-20T03:00:00", "date_ranges": [
|
||||
{"column": "order_date", "min": "2025-01-01", "max": "2026-10-19"}]}}}
|
||||
base, target = period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19")
|
||||
fallback = await compare(base, target, probe=fixed_prober({}), facts=facts, measure="gmv")
|
||||
assert fallback["verdict"]["code"] == "comparable" and fallback["phrase"] == "两期可比"
|
||||
assert fallback["probes"] == [{"source": "shop", "table": "orders", "column": "order_date", "min": "2025-01-01",
|
||||
"max": "2026-10-19", "basis": "profile", "at": "2026-10-20"}]
|
||||
nothing = await compare(base, target, measure="gmv")
|
||||
assert nothing["verdict"]["code"] == "comparable"
|
||||
assert nothing["phrase"] == "两期可比，整期核对不了" and nothing["probes"] == []
|
||||
|
||||
|
||||
async def test_bounds_computed_from_the_run_day_are_evaluated_in_the_database_and_recorded():
|
||||
sql = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', '{a}') AND order_date < date('now', '{b}')"
|
||||
base = snap("Q1", sql.format(a="-16 days", b="-9 days"), ["gmv"], [[100]])
|
||||
target = snap("Q2", sql.format(a="-9 days", b="-2 days"), ["gmv"], [[90]])
|
||||
bounds = {"DATE('now', '-16 days')": "2026-10-05", "DATE('now', '-9 days')": "2026-10-12",
|
||||
"DATE('now', '-2 days')": "2026-10-19"}
|
||||
probe = fixed_prober(RANGE, bounds=bounds)
|
||||
result = await compare(base, target, probe=probe, measure="gmv")
|
||||
assert result["verdict"]["code"] == "comparable"
|
||||
assert checks(result)["同口径"] == "通过" # 只差按运行当天现算的界
|
||||
assert result["values"]["base_start"] == "2026-10-05" and result["values"]["target_end"] == "2026-10-18"
|
||||
exprs = [p["expr"] for p in result["probes"] if "expr" in p]
|
||||
assert exprs == ["DATE('now', '-16 days')", "DATE('now', '-9 days')", "DATE('now', '-2 days')"]
|
||||
assert all(p["basis"] == "live" for p in result["probes"])
|
||||
# 界算不出来：窗口核对不了，不判不可比
|
||||
blind = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert blind["verdict"]["code"] == "comparable" and "窗口长度" in blind["phrase"]
|
||||
|
||||
|
||||
async def test_data_problems_on_the_snapshots():
|
||||
base = period("Q1", "2026-10-05", "2026-10-12")
|
||||
empty = period("Q2", "2026-10-12", "2026-10-19", no_data=[{"columns": ["gmv"], "table": "orders",
|
||||
"column": "order_date", "message": "没有数据"}])
|
||||
result = await compare(base, empty, probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["verdict"]["code"] == "incomparable" and "本期查询的区间里这张表没有数据" in result["phrase"]
|
||||
checked = period("Q2", "2026-10-12", "2026-10-19", checks=[{"code": "fanout_sum", "level": "error",
|
||||
"message": "关联订单明细后对订单金额求和，会重复计算"}])
|
||||
warned = await compare(base, checked, probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert warned["verdict"]["code"] == "conditional"
|
||||
assert "本期查询的 SQL 检查有错误：关联订单明细后对订单金额求和，会重复计算" in warned["phrase"]
|
||||
|
||||
|
||||
async def test_one_snapshot_grouped_by_period_is_naturally_the_same_query():
|
||||
sql = "SELECT order_date AS day, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-01' GROUP BY 1"
|
||||
rows = [[f"2026-10-{d:02d}", 100] for d in range(1, 20)]
|
||||
s = snap("Q1", sql, ["day", "gmv"], rows, types={"day": "text", "gmv": "number"})
|
||||
probe = fixed_prober(RANGE)
|
||||
same_weekday = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="day", base="2026-10-05",
|
||||
target="2026-10-12")
|
||||
assert same_weekday["verdict"]["code"] == "comparable" and same_weekday["params"]["measure"] == "gmv"
|
||||
assert same_weekday["params"]["grain"] == "day"
|
||||
weekend = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="day", base="2026-10-09",
|
||||
target="2026-10-10")
|
||||
assert weekend["phrase"] == "两期有条件可比：两段的周末天数不同：0 天对 1 天"
|
||||
# 期次有一截落在 SQL 的时间条件之外：基期被截掉了；本期（10 月）还没过完
|
||||
monthly_rows = [["2026-09-01", 100], ["2026-10-01", 90]]
|
||||
month = snap("Q1", "SELECT DATE(order_date, 'start of month') AS month, SUM(amount) AS gmv FROM orders "
|
||||
"WHERE order_date >= '2026-09-05' GROUP BY 1", ["month", "gmv"], monthly_rows)
|
||||
partial = await analyze("comparable", snaps=[month], probe=probe, query="Q1", period="month", base="2026-09-01",
|
||||
target="2026-10-01", measure="gmv")
|
||||
assert partial["verdict"]["code"] == "incomparable"
|
||||
assert "基期数据从 2026-09-05 才开始，缺前 4 天" in partial["phrase"]
|
||||
assert "本期数据只到 2026-10-20，还差 11 天" in partial["phrase"]
|
||||
|
||||
|
||||
async def test_reasons_are_split_one_sentence_each():
|
||||
result = await compare(period("Q1", "2026-10-05", "2026-10-08"), period("Q2", "2026-10-16", "2026-10-21"),
|
||||
probe=fixed_prober(RANGE), measure="gmv")
|
||||
reasons = comparable_reasons(result)
|
||||
assert "两段的天数不同：3 天对 5 天" in reasons and "本期数据只到 2026-10-20，还差 0 天" not in reasons
|
||||
assert all("；" not in r for r in reasons)
|
||||
assert comparable_reasons({"table": None}) == []
|
||||
|
||||
|
||||
async def test_a_masked_date_column_is_neither_used_nor_probed():
|
||||
probe = fixed_prober(RANGE)
|
||||
result = await compare(period("Q1", "2026-10-05", "2026-10-12", masked=["order_date"]),
|
||||
period("Q2", "2026-10-12", "2026-10-19", masked=["order_date"]), probe=probe, measure="gmv")
|
||||
assert probe.calls == [] and result["probes"] == []
|
||||
assert result["verdict"]["code"] == "comparable" and "窗口长度" in result["phrase"]
|
||||
@@ -0,0 +1,420 @@
|
||||
"""变化拆解（app/analysis/decompose.py，细案 01 §2）：三种恒等式逐位成立、辛普森、抵消、不闭合、下钻闭合与超深、新出现 /
|
||||
消失的段、比率当可加的调用错误……每个用例都顺带核对复算和运行逐字段相同（tests/analysis_samples.analyze）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import random
|
||||
from decimal import Decimal
|
||||
from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
|
||||
from analysis_samples import analyze, call_error, fixed_prober, notes, snap
|
||||
from app.analysis.decompose import lmdi, ratio_effects
|
||||
|
||||
SQL = "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week"
|
||||
TYPES = {"region": "text", "week": "text", "gmv": "number"}
|
||||
WEEKLY = [["华东", "2026-W41", 820], ["华东", "2026-W42", 700], ["华北", "2026-W41", 600], ["华北", "2026-W42", 560],
|
||||
["华南", "2026-W41", 700], ["华南", "2026-W42", 666], ["西南", "2026-W41", 100], ["西南", "2026-W42", 100]]
|
||||
|
||||
|
||||
def weekly(rows=WEEKLY, **kw):
|
||||
return snap("Q2", SQL, ["region", "week", "gmv"], rows, types=TYPES, **kw)
|
||||
|
||||
|
||||
async def additive(rows=WEEKLY, *, snaps=None, **params):
|
||||
params = {"query": "Q2", "by": ["region"], "period": "week", "base": "2026-W41", "target": "2026-W42",
|
||||
"measure": "gmv", **params}
|
||||
return await analyze("decompose", "additive", snaps=snaps or [weekly(rows)], **params)
|
||||
|
||||
|
||||
def _rows(base: dict, target: dict) -> list[list]:
|
||||
return [[k, "2026-W41", v] for k, v in base.items()] + [[k, "2026-W42", v] for k, v in target.items()]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# additive
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_driver_is_named_with_its_share_and_the_whole_shape_is_filled():
|
||||
result = await additive(label="销售额", hypothesis="华东拖累")
|
||||
assert result["method"] == "decompose.additive@1" and result["status"] == "ok"
|
||||
assert result["verdict"] == {"code": "driver", "subjects": [["华东"]], "direction": "down"}
|
||||
assert result["phrase"] == "销售额下降 194（-8.7%），主要来自华东（-120，占 61.9%）"
|
||||
assert result["values"]["delta_total"] == "-194" and result["values"]["top_key"] == "华东"
|
||||
assert result["values"]["top_share"] == "0.618556701031" and result["values"]["segments"] == "4"
|
||||
table = result["table"]
|
||||
assert table["columns"] == ["region", "base", "target", "delta", "pct", "share", "flag"]
|
||||
assert table["labels"]["region"] == "地区" and table["labels"]["share"] == "占变化的比重"
|
||||
assert [r[0] for r in table["rows"]] == ["华东", "华北", "华南", "西南"] # 按变化的绝对值从大到小
|
||||
assert result["settings"] == {"main_share": 0.5, "concentration": 0.8, "offset_ratio": 0.25, "flat_pct": 0.005,
|
||||
"table_top_k": 10}
|
||||
assert result["inputs"] == [{"ref": "Q2", "artifact": weekly()["artifact"], "role": "data",
|
||||
"columns": ["region", "week", "gmv"], "rows": "all"}]
|
||||
assert result["hypothesis"] == "华东拖累" and result["probes"] == []
|
||||
assert result["criterion"].endswith("以各段之和为总数")
|
||||
assert {c["code"] for c in result["caveats"]} == {"headline_basis"}
|
||||
assert result["params"]["catalog"] == {"labels": {"region": "地区"}, "measures": {"gmv": "flow"}}
|
||||
|
||||
|
||||
async def test_the_segments_add_up_to_the_total_digit_for_digit():
|
||||
rng = random.Random(7)
|
||||
regions = ["华东", "华南", "华北", "西南", "东北", "西北"]
|
||||
for _ in range(20):
|
||||
base = {r: round(rng.uniform(0, 5000), rng.choice([0, 2, 3])) for r in regions}
|
||||
target = {r: round(rng.uniform(0, 5000), rng.choice([0, 2, 3])) for r in regions}
|
||||
result = await additive(_rows(base, target))
|
||||
rows = result["table"]["rows"]
|
||||
assert sum(Decimal(r[3]) for r in rows) == Decimal(result["values"]["delta_total"])
|
||||
assert sum(Decimal(r[1]) for r in rows) == Decimal(result["values"]["base_total"])
|
||||
assert sum(Decimal(r[2]) for r in rows) == Decimal(result["values"]["target_total"])
|
||||
assert Decimal(result["values"]["target_total"]) - Decimal(result["values"]["base_total"]) == \
|
||||
Decimal(result["values"]["delta_total"])
|
||||
assert result["values"]["gross_up"] and result["values"]["gross_down"]
|
||||
|
||||
|
||||
async def test_concentrated_and_dispersed():
|
||||
base = {f"店{i}": 1000 for i in range(10)}
|
||||
target = {**base, "店0": 1450, "店1": 1400, "店2": 1050, "店3": 1050, "店4": 1050}
|
||||
result = await additive(_rows(base, target), by="region")
|
||||
assert result["verdict"]["code"] == "concentrated"
|
||||
assert result["verdict"]["subjects"] == [["店0"], ["店1"]]
|
||||
assert result["values"]["top_k"] == "2" and result["values"]["top_k_share"] == "0.85"
|
||||
assert "集中在店0、店1（合计占 85.0%）" in result["phrase"]
|
||||
spread = await additive(_rows({r: 1000 for r in "甲乙丙丁戊"}, {r: 1100 for r in "甲乙丙丁戊"}))
|
||||
assert spread["verdict"] == {"code": "dispersed", "subjects": [], "direction": "up"}
|
||||
assert "分散在各地区，没有哪一段占到一半" in spread["phrase"]
|
||||
|
||||
|
||||
async def test_offsetting_segments_are_said_and_a_share_can_pass_one_hundred_percent():
|
||||
result = await additive(_rows({"华东": 1000, "华北": 500, "华南": 500}, {"华东": 850, "华北": 540, "华南": 490}))
|
||||
assert result["values"]["delta_total"] == "-120"
|
||||
assert result["verdict"]["code"] == "driver" and result["values"]["top_share"] == "1.25"
|
||||
offset = next(c for c in result["caveats"] if c["code"] == "offset")
|
||||
assert offset["text"] == "下降的段合计 -160，上升的段合计 +40，互相抵消了一部分"
|
||||
assert result["phrase"] == "销售额下降 120（-6.0%），主要来自华东（-150，占 125.0%）；下降的段合计 -160，上升的段合计 +40，互相抵消了一部分"
|
||||
|
||||
|
||||
async def test_flat_with_offsets_says_what_moved_inside():
|
||||
result = await additive(_rows({"华东": 1000, "华北": 1000, "华南": 220}, {"华东": 1120, "华北": 882, "华南": 220}))
|
||||
assert result["verdict"] == {"code": "flat", "subjects": [], "direction": None}
|
||||
assert result["phrase"] == "销售额基本持平（+0.1%），内部此消彼长：华东 +120、华北 -118"
|
||||
|
||||
|
||||
async def test_a_headline_that_does_not_close_stops_the_concentration_verdict():
|
||||
head = snap("Q1", "SELECT week, SUM(amount) AS gmv FROM orders GROUP BY week", ["week", "gmv"],
|
||||
[["2026-W42", 2050], ["2026-W41", 2220]], types={"week": "text", "gmv": "number"})
|
||||
result = await additive(snaps=[weekly(), head], headline={"base": "Q1.r1.gmv", "target": "Q1.r0.gmv"})
|
||||
assert result["verdict"] == {"code": "unclosed", "subjects": [], "direction": None}
|
||||
assert result["values"]["gap"] == "24" and result["values"]["top_share"] is None
|
||||
assert all(r[5] is None for r in result["table"]["rows"]) # 不给占比
|
||||
assert result["phrase"] == "按地区拆开后各段之和和总数对不上（差 24），不判断变化来自哪里"
|
||||
assert [i["role"] for i in result["inputs"]] == ["headline_base", "headline_target", "data"]
|
||||
assert result["inputs"][0] == {"ref": "Q1.r1.gmv", "artifact": head["artifact"], "role": "headline_base",
|
||||
"row": 1, "column": "gmv", "value": "2220"}
|
||||
assert "各段之和要和总数对得上" in result["criterion"]
|
||||
|
||||
|
||||
async def test_a_rounded_headline_closes_within_its_written_precision():
|
||||
rows = [["华东", "2026-W41", "820.25"], ["华东", "2026-W42", "700.12"], ["华北", "2026-W41", "1399.9"],
|
||||
["华北", "2026-W42", "1325.84"]]
|
||||
for written, closes in (("2220.2", True), ("2220", True), ("2221", False)):
|
||||
head = snap("Q1", "SELECT SUM(amount) AS gmv FROM orders", ["gmv_base", "gmv"], [[written, "2025.96"]])
|
||||
result = await additive(rows, snaps=[weekly(rows), head], headline=["Q1.r0.gmv_base", "Q1.r0.gmv"])
|
||||
assert (result["verdict"]["code"] != "unclosed") is closes, written
|
||||
|
||||
|
||||
async def test_drilling_down_checks_closure_against_the_parent_segment():
|
||||
top = await additive()
|
||||
province = snap("Q4", "SELECT province, week, SUM(amount) AS gmv FROM orders WHERE region = '华东' GROUP BY 1, 2",
|
||||
["province", "week", "gmv"],
|
||||
[["沪", "2026-W41", 500], ["沪", "2026-W42", 410], ["苏", "2026-W41", 320], ["苏", "2026-W42", 290]])
|
||||
params = {"query": "Q4", "by": ["province"], "period": "week", "base": "2026-W41", "target": "2026-W42",
|
||||
"measure": "gmv", "parent": {"of": "A3", "key": ["华东"]}}
|
||||
child = await analyze("decompose", "additive", snaps=[province], parents={"A3": top}, **params)
|
||||
assert child["verdict"]["code"] == "driver" and child["verdict"]["subjects"] == [["沪"]]
|
||||
assert child["params"]["depth"] == 1 and child["params"]["parent"] == {"of": "A3", "key": ["华东"]}
|
||||
assert child["inputs"][0]["role"] == "parent" and child["inputs"][0]["ref"] == "A3"
|
||||
assert "max_drill_depth" in child["settings"]
|
||||
assert {c["code"] for c in child["caveats"]} == set() # 有 parent 就不是「以各段之和为总数」
|
||||
short = snap("Q4", province["sql"], ["province", "week", "gmv"],
|
||||
[["沪", "2026-W41", 500], ["沪", "2026-W42", 410], ["苏", "2026-W41", 300], ["苏", "2026-W42", 290]])
|
||||
unclosed = await analyze("decompose", "additive", snaps=[short], parents={"A3": top}, **params)
|
||||
assert unclosed["verdict"]["code"] == "unclosed" and unclosed["values"]["gap"] == "20"
|
||||
# 超深：上一层已经是第 1 层，最多 1 层
|
||||
error = await call_error("decompose", "additive", snaps=[province], parents={"A3": child},
|
||||
settings={"max_drill_depth": 1}, **{**params, "parent": {"of": "A3", "key": ["沪"]}})
|
||||
assert error.code == "too_deep" and "最多 1 层" in error.message
|
||||
missing = await call_error("decompose", "additive", snaps=[province], parents={"A3": top},
|
||||
**{**params, "parent": {"of": "A3", "key": ["东北"]}})
|
||||
assert missing.code == "parent_key_missing"
|
||||
absent = await call_error("decompose", "additive", snaps=[province], **params)
|
||||
assert absent.code == "parent_missing"
|
||||
|
||||
|
||||
async def test_new_and_vanished_segments_and_empty_cells_are_flagged():
|
||||
rows = [["华东", "2026-W41", 800], ["华东", "2026-W42", 700], ["西南", "2026-W42", 90],
|
||||
["东北", "2026-W41", 40], ["华北", "2026-W41", None], ["华北", "2026-W42", 30]]
|
||||
result = await additive(rows)
|
||||
flags = {r[0]: r[6] for r in result["table"]["rows"]}
|
||||
assert flags == {"华东": None, "西南": "新出现", "东北": "消失", "华北": "为空"}
|
||||
null = next(c for c in result["caveats"] if c["code"] == "null_values")
|
||||
assert null["text"] == "华北的销售额为空，按 0 算"
|
||||
assert next(r for r in result["table"]["rows"] if r[0] == "西南")[4] is None # 基期为 0 不给变化率
|
||||
|
||||
|
||||
async def test_two_dimension_keys():
|
||||
sql = "SELECT region, channel, week, SUM(amount) AS gmv FROM orders GROUP BY 1, 2, 3"
|
||||
rows = [["华东", "线上", "W1", 500], ["华东", "线上", "W2", 300], ["华东", "线下", "W1", 300], ["华东", "线下", "W2", 310],
|
||||
["华北", "线上", "W1", 200], ["华北", "线上", "W2", 205]]
|
||||
s = snap("Q2", sql, ["region", "channel", "week", "gmv"], rows)
|
||||
result = await analyze("decompose", "additive", snaps=[s], query="Q2", by=["region", "channel"], period="week",
|
||||
base="W1", target="W2", measure="gmv")
|
||||
assert result["verdict"]["subjects"] == [["华东", "线上"]]
|
||||
assert "主要来自华东·线上" in result["phrase"]
|
||||
|
||||
|
||||
async def test_ratio_and_non_additive_measures_are_call_errors():
|
||||
avg = snap("Q2", "SELECT region, week, AVG(amount) AS aov FROM orders GROUP BY 1, 2", ["region", "week", "aov"], WEEKLY)
|
||||
error = await call_error("decompose", "additive", snaps=[avg], query="Q2", by="region", period="week",
|
||||
base="2026-W41", target="2026-W42", measure="aov")
|
||||
assert error.code == "ratio_as_additive" and "改用比率拆解" in error.message
|
||||
rate_sql = "SELECT region, week, SUM(refund_rate) AS r FROM orders GROUP BY 1, 2"
|
||||
labeled = {"shop": {"orders": notes(columns={"refund_rate": {"measure": "ratio"}})}}
|
||||
error = await call_error("decompose", "additive", snaps=[snap("Q2", rate_sql, ["region", "week", "r"], WEEKLY)],
|
||||
catalogs=labeled, query="Q2", by="region", period="week", base="2026-W41",
|
||||
target="2026-W42", measure="r")
|
||||
assert error.code == "ratio_as_additive"
|
||||
# 只是推断（没人确认）的照算，写附带说明
|
||||
guessed = {"shop": {"orders": notes(columns={"refund_rate": {"measure": {"value": "ratio", "source": "llm",
|
||||
"status": "proposed"}}})}}
|
||||
result = await analyze("decompose", "additive", snaps=[snap("Q2", rate_sql, ["region", "week", "r"], WEEKLY)],
|
||||
catalogs=guessed, query="Q2", by="region", period="week", base="2026-W41",
|
||||
target="2026-W42", measure="r")
|
||||
note = next(c for c in result["caveats"] if c["code"] == "measure_unlabeled")
|
||||
assert note["text"] == "目录里「r」是比率只是推断，没人确认，按可以按段相加处理"
|
||||
unlabeled = await additive(snaps=[weekly()], catalogs={})
|
||||
assert any(c["code"] == "measure_unlabeled" and "目录没标「gmv」的度量类型" in c["text"] for c in unlabeled["caveats"])
|
||||
|
||||
|
||||
async def test_call_errors_tell_the_agent_how_to_fix_the_call():
|
||||
dup = WEEKLY + [["华东", "2026-W42", 5]]
|
||||
error = await call_error("decompose", "additive", snaps=[weekly(dup)], query="Q2", by="region", period="week",
|
||||
base="2026-W41", target="2026-W42", measure="gmv")
|
||||
assert error.code == "duplicate_rows" and "在 SQL 里按 地区、week 汇总" in error.message
|
||||
masked = await call_error("decompose", "additive", snaps=[weekly(masked=["Region"])], query="Q2", by="region",
|
||||
period="week", base="2026-W41", target="2026-W42", measure="gmv")
|
||||
assert masked.code == "masked_column"
|
||||
missing = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="region", period="week",
|
||||
base="2026-W40", target="2026-W42", measure="gmv")
|
||||
assert missing.code == "period_not_found" and "2026-W41" in missing.message
|
||||
column = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="area", period="week",
|
||||
base="2026-W41", target="2026-W42", measure="gmv")
|
||||
assert column.code == "unknown_column" and "region、week、gmv" in column.message
|
||||
text = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="gmv", period="week",
|
||||
base="2026-W41", target="2026-W42", measure="region")
|
||||
assert text.code == "not_numeric"
|
||||
unknown = await call_error("decompose", "additive", snaps=[weekly()], query="Q9", by="region", period="week",
|
||||
base="2026-W41", target="2026-W42", measure="gmv")
|
||||
assert unknown.code == "unknown_query"
|
||||
same_column = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="week", period="week",
|
||||
base="2026-W41", target="2026-W42", measure="gmv")
|
||||
assert same_column.code == "bad_by"
|
||||
|
||||
|
||||
async def test_truncated_snapshots_and_single_segments_are_refused():
|
||||
result = await additive(snaps=[weekly(truncated=True)])
|
||||
assert result["status"] == "refused" and result["reason_code"] == "truncated"
|
||||
assert result["reason"] == "Q2 只取回了前 8 行，各段之和不是完整的和"
|
||||
assert result["phrase"] == "变化拆解没有做成：Q2 只取回了前 8 行，各段之和不是完整的和"
|
||||
assert result["verdict"] is None and result["table"] is None and result["values"] == {}
|
||||
one = await additive([["华东", "2026-W41", 1], ["华东", "2026-W42", 2]])
|
||||
assert one["reason_code"] == "too_few_segments"
|
||||
|
||||
|
||||
async def test_two_snapshots_check_comparability_first():
|
||||
base = snap("Q1", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' AND "
|
||||
"order_date < '2026-10-12' GROUP BY region", ["region", "gmv"],
|
||||
[["华东", 820], ["华北", 600], ["华南", 700]])
|
||||
target = snap("Q2", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-12' AND "
|
||||
"order_date < '2026-10-19' GROUP BY region", ["region", "gmv"],
|
||||
[["华东", 700], ["华北", 560], ["华南", 666]])
|
||||
probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")})
|
||||
result = await analyze("decompose", "additive", snaps=[base, target], probe=probe, base_query="Q1",
|
||||
target_query="Q2", by="region", measure="gmv")
|
||||
assert result["verdict"]["code"] == "driver" and result["params"]["today"] == "2026-10-21"
|
||||
assert not any(c["code"] == "comparability" for c in result["caveats"])
|
||||
assert [i["role"] for i in result["inputs"]] == ["base", "target"]
|
||||
assert result["probes"] == [{"source": "shop", "table": "orders", "column": "order_date", "min": "2025-01-01",
|
||||
"max": "2026-10-20", "basis": "live", "at": "2026-10-21T09:00:00"}]
|
||||
# 本期没过完：照样拆，判定后附上可比性结论
|
||||
partial = snap("Q2", target["sql"].replace("'2026-10-19'", "'2026-10-25'"), ["region", "gmv"], target["rows"])
|
||||
result = await analyze("decompose", "additive", snaps=[base, partial], probe=probe, base_query="Q1",
|
||||
target_query="Q2", by="region", measure="gmv")
|
||||
note = next(c for c in result["caveats"] if c["code"] == "comparability")
|
||||
assert note["text"].startswith("两期不可比：") and "本期数据只到 2026-10-20，还差 4 天" in note["text"]
|
||||
assert result["verdict"]["code"] == "driver" and note["text"] in result["phrase"]
|
||||
# 口径结构不同：不拆
|
||||
other = snap("Q2", "SELECT region, SUM(amount) AS gmv FROM orders JOIN refunds USING (order_id) WHERE order_date "
|
||||
">= '2026-10-12' GROUP BY region", ["region", "gmv"], target["rows"])
|
||||
refused = await analyze("decompose", "additive", snaps=[base, other], probe=probe, base_query="Q1",
|
||||
target_query="Q2", by="region", measure="gmv")
|
||||
assert refused["reason_code"] == "incomparable_sides" and refused["reason"].startswith("两份查询的口径结构不同")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# ratio
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
RATIO_SQL = "SELECT channel, week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY 1, 2"
|
||||
|
||||
|
||||
def _ratio_snap(rows):
|
||||
return snap("Q1", RATIO_SQL, ["channel", "week", "refunds", "orders"], rows)
|
||||
|
||||
|
||||
async def ratio(rows, **params):
|
||||
params = {"query": "Q1", "by": "channel", "period": "week", "base": "W41", "target": "W42",
|
||||
"numerator": "refunds", "denominator": "orders", **params}
|
||||
return await analyze("decompose", "ratio", snaps=[_ratio_snap(rows)], **params)
|
||||
|
||||
|
||||
def test_mix_plus_rate_effects_equal_the_ratio_change_exactly():
|
||||
rng = random.Random(11)
|
||||
for _ in range(200):
|
||||
segs = []
|
||||
for _ in range(rng.randint(2, 7)):
|
||||
d_b, d_t = rng.choice([0, rng.randint(1, 5000)]), rng.randint(1, 5000)
|
||||
segs.append((Fraction(rng.randint(0, d_b)) if d_b else Fraction(0), Fraction(d_b),
|
||||
Fraction(rng.randint(0, d_t)), Fraction(d_t)))
|
||||
if sum(s[1] for s in segs) == 0:
|
||||
continue
|
||||
R, R2, effects = ratio_effects(segs)
|
||||
assert sum((e[4] for e in effects), Fraction(0)) + sum((e[5] for e in effects), Fraction(0)) == R2 - R
|
||||
assert sum((e[0] for e in effects), Fraction(0)) == 1 == sum((e[1] for e in effects), Fraction(0))
|
||||
|
||||
|
||||
async def test_simpson_each_channel_improves_but_the_mix_makes_the_whole_worse():
|
||||
rows = [["线上", "W41", 50, 500], ["线上", "W42", 90, 1000], ["线下", "W41", 10, 1000], ["线下", "W42", 4, 500]]
|
||||
result = await ratio(rows, label="退款率")
|
||||
assert result["verdict"] == {"code": "mix_driven", "subjects": [["线上"]], "direction": "up"}
|
||||
assert result["phrase"].startswith("退款率上升 2.3 个百分点（4.0% 到 6.3%），主要来自结构变化：线上占比从 33.3% 到 66.7%")
|
||||
simpson = next(c for c in result["caveats"] if c["code"] == "simpson")
|
||||
assert simpson["text"] == "各段的退款率大多在下降，整体却上升：变化来自结构，线上占比从 33.3% 升到 66.7%"
|
||||
values = result["values"]
|
||||
assert Fraction(values["mix_total"]) + Fraction(values["rate_total"]) == pytest.approx(Fraction(values["delta"]))
|
||||
assert result["table"]["columns"][:5] == ["channel", "n_base", "n_target", "d_base", "d_target"]
|
||||
|
||||
|
||||
async def test_rate_driven_both_and_small_denominators():
|
||||
rows = [["线上", "W41", 50, 1000], ["线上", "W42", 80, 1000], ["线下", "W41", 20, 1000], ["线下", "W42", 26, 1000]]
|
||||
result = await ratio(rows, label="退款率")
|
||||
assert result["verdict"]["code"] == "rate_driven" and result["verdict"]["subjects"] == [["线上"]]
|
||||
assert result["phrase"] == "退款率上升 1.8 个百分点（3.5% 到 5.3%），主要来自各段自身的变化，线上影响最大"
|
||||
tiny = [["线上", "W41", 2, 20], ["线上", "W42", 3, 25], ["线下", "W41", 20, 1000], ["线下", "W42", 30, 1000]]
|
||||
small = await ratio(tiny, label="退款率")
|
||||
note = next(c for c in small["caveats"] if c["code"] == "small_denominator")
|
||||
assert note["text"] == "线上的 orders 不到 30，比率波动大，不参与多数判断"
|
||||
assert "分母小" in next(r for r in small["table"]["rows"] if r[0] == "线上")[-1]
|
||||
|
||||
|
||||
async def test_ratio_refusals():
|
||||
zero = [["线上", "W41", 0, 0], ["线上", "W42", 3, 100], ["线下", "W41", 0, 0], ["线下", "W42", 2, 100]]
|
||||
result = await ratio(zero)
|
||||
assert result["reason_code"] == "zero_base" and result["reason"] == "基期的 orders 合计为 0，算不了比率"
|
||||
orphan = [["线上", "W41", 5, 0], ["线上", "W42", 3, 100], ["线下", "W41", 1, 100], ["线下", "W42", 2, 100]]
|
||||
assert (await ratio(orphan))["reason_code"] == "zero_base"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# factor
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _factor_snap(rows):
|
||||
return snap("Q1", "SELECT week, SUM(amount) AS gmv, COUNT(*) AS orders, ROUND(AVG(amount), 2) AS aov FROM orders "
|
||||
"GROUP BY week", ["week", "gmv", "orders", "aov"], rows)
|
||||
|
||||
|
||||
async def factor(rows, **params):
|
||||
params = {"query": "Q1", "period": "week", "base": "W41", "target": "W42", "product": "gmv",
|
||||
"factors": ["orders", "aov"], "label": "销售额", **params}
|
||||
return await analyze("decompose", "factor", snaps=[_factor_snap(rows)], **params)
|
||||
|
||||
|
||||
def test_lmdi_contributions_add_up_to_the_change_within_float_error():
|
||||
rng = random.Random(3)
|
||||
for _ in range(200):
|
||||
k = rng.randint(2, 4)
|
||||
base = [rng.uniform(0.5, 500) for _ in range(k)]
|
||||
target = [b * rng.uniform(0.5, 1.8) for b in base]
|
||||
m_b = math.prod(base) * (1 + rng.uniform(-1e-6, 1e-6)) # 取整过的因子：乘积差一点点
|
||||
m_t = math.prod(target) * (1 + rng.uniform(-1e-6, 1e-6))
|
||||
contrib = lmdi(base, target, m_b, m_t)
|
||||
assert math.fsum(contrib) == pytest.approx(m_t - m_b, rel=1e-9, abs=1e-9 * max(m_b, m_t))
|
||||
same = lmdi([2.0, 3.0], [2.0, 3.0], 6.0, 6.0)
|
||||
assert same == [0.0, 0.0]
|
||||
|
||||
|
||||
async def test_factor_driver_with_rounded_factors():
|
||||
rows = [["W41", 1000, 137, "7.30"], ["W42", 1320, 160, "8.25"]]
|
||||
result = await factor(rows)
|
||||
assert result["verdict"]["code"] in ("driver", "both")
|
||||
contrib = [Fraction(r[4]) for r in result["table"]["rows"]]
|
||||
assert float(sum(contrib)) == pytest.approx(320, rel=1e-9)
|
||||
clean = await factor([["W41", "1000", 100, "10"], ["W42", "1320", 120, "11"]])
|
||||
assert clean["verdict"] == {"code": "driver", "subjects": [["orders"]], "direction": "up"}
|
||||
assert clean["phrase"].startswith("销售额上升 320（+32.0%），主要来自 orders 的变化（贡献 +")
|
||||
assert clean["table"]["columns"] == ["factor", "base", "target", "pct", "contrib", "share"]
|
||||
|
||||
|
||||
async def test_factor_call_errors_and_refusals():
|
||||
# 客单价写到两位小数：容差是它的半个末位（加上订单数的），差 1.5% 不是取整造成的
|
||||
error = await call_error("decompose", "factor", snaps=[_factor_snap([["W41", 1000, 100, "10.00"],
|
||||
["W42", 1300, 120, "11.00"]])],
|
||||
query="Q1", period="week", base="W41", target="W42", product="gmv",
|
||||
factors=["orders", "aov"])
|
||||
assert error.code == "not_product" and "本期差" in error.message
|
||||
negative = await factor([["W41", 0, 0, 10], ["W42", 1320, 120, 11]])
|
||||
assert negative["reason_code"] == "non_positive"
|
||||
flat = await factor([["W41", 1000, 100, 10], ["W42", 1000, 125, 8]])
|
||||
assert flat["verdict"]["code"] == "flat"
|
||||
few = await call_error("decompose", "factor", snaps=[_factor_snap([["W41", 1, 1, 1], ["W42", 1, 1, 1]])],
|
||||
query="Q1", period="week", base="W41", target="W42", product="gmv", factors=["orders"])
|
||||
assert few.code == "bad_factors"
|
||||
|
||||
|
||||
async def test_ratio_headline_closure_and_drilling_into_a_channel():
|
||||
rows = [["线上", "W41", 50, 1000], ["线上", "W42", 80, 1000], ["线下", "W41", 20, 1000], ["线下", "W42", 26, 1000]]
|
||||
head = snap("Q9", "SELECT week, SUM(refund_flag) * 1.0 / COUNT(*) AS rate FROM orders GROUP BY week",
|
||||
["week", "rate"], [["W41", "0.035"], ["W42", "0.053"]])
|
||||
params = {"query": "Q1", "by": "channel", "period": "week", "base": "W41", "target": "W42",
|
||||
"numerator": "refunds", "denominator": "orders", "label": "退款率"}
|
||||
closed = await analyze("decompose", "ratio", snaps=[_ratio_snap(rows), head],
|
||||
headline={"base": "Q9.r0.rate", "target": "Q9.r1.rate"}, **params)
|
||||
assert closed["verdict"]["code"] == "rate_driven"
|
||||
off = snap("Q9", head["sql"], ["week", "rate"], [["W41", "0.035"], ["W42", "0.06"]])
|
||||
unclosed = await analyze("decompose", "ratio", snaps=[_ratio_snap(rows), off],
|
||||
headline={"base": "Q9.r0.rate", "target": "Q9.r1.rate"}, **params)
|
||||
assert unclosed["verdict"]["code"] == "unclosed"
|
||||
assert unclosed["phrase"].startswith("按渠道拆开后各段合起来的比率和总数对不上")
|
||||
# 线上按地区下钻：各地区的分子、分母之和要等于上一层线上那一段的
|
||||
region = snap("Q2", "SELECT region, week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders "
|
||||
"WHERE channel = '线上' GROUP BY 1, 2", ["region", "week", "refunds", "orders"],
|
||||
[["华东", "W41", 30, 600], ["华东", "W42", 55, 600], ["华北", "W41", 20, 400], ["华北", "W42", 25, 400]])
|
||||
child = await analyze("decompose", "ratio", snaps=[region], parents={"A1": closed},
|
||||
**{**params, "query": "Q2", "by": "region", "parent": {"of": "A1", "key": ["线上"]}})
|
||||
assert child["verdict"]["code"] != "unclosed" and child["params"]["depth"] == 1
|
||||
wrong = await call_error("decompose", "additive", snaps=[weekly()], parents={"A1": closed}, query="Q2",
|
||||
by="region", period="week", base="2026-W41", target="2026-W42", measure="gmv",
|
||||
parent={"of": "A1", "key": ["线上"]})
|
||||
assert wrong.code == "parent_not_decompose"
|
||||
|
||||
|
||||
async def test_the_same_query_cannot_be_both_periods():
|
||||
error = await call_error("decompose", "additive", snaps=[weekly()], base_query="Q2", target_query="Q2",
|
||||
by="region", measure="gmv")
|
||||
assert error.code == "same_query"
|
||||
error = await call_error("comparable", snaps=[weekly()], base_query="Q2", target_query="Q2")
|
||||
assert error.code == "same_query"
|
||||
@@ -0,0 +1,138 @@
|
||||
"""差异检验（app/analysis/differ.py，细案 01 §5）：比率小样本走 Fisher、均值 Welch、Holm、rows 敏感性翻转。每个用例都顺带核对
|
||||
复算和运行逐字段相同。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
|
||||
from analysis_samples import analyze, call_error, snap
|
||||
from app.analysis import stats
|
||||
|
||||
RATE_SQL = "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region"
|
||||
|
||||
|
||||
def rates(rows):
|
||||
return snap("Q1", RATE_SQL, ["region", "refunds", "orders"], rows)
|
||||
|
||||
|
||||
async def rate(rows, **params):
|
||||
params = {"query": "Q1", "by": "region", "numerator": "refunds", "denominator": "orders", "label": "退款率", **params}
|
||||
return await analyze("differ", "rate", snaps=[rates(rows)], **params)
|
||||
|
||||
|
||||
async def test_small_samples_use_fisher_and_say_it_may_be_noise():
|
||||
result = await rate([["西南", 2, 40], ["华东", 5, 230]])
|
||||
assert result["method"] == "differ.rate@1"
|
||||
assert result["verdict"] == {"code": "no_evidence", "subjects": [["西南"], ["华东"]], "direction": None}
|
||||
assert result["values"]["test"] == "fisher" and result["values"]["n_a"] == "40" and result["values"]["n_b"] == "230"
|
||||
assert Fraction(result["values"]["p"]) == pytest.approx(stats.fisher_exact(2, 38, 5, 225), rel=1e-9)
|
||||
assert result["phrase"] == ("西南和华东的退款率差异可能只是样本波动（样本量 40、230）；样本少（有格子的期望频数不到 5），"
|
||||
"用了精确检验")
|
||||
assert "p 值" not in result["phrase"] and "显著" not in result["phrase"]
|
||||
assert result["settings"] == {"differ_alpha": 0.05, "min_expected": 5}
|
||||
|
||||
|
||||
async def test_large_samples_use_the_pooled_z_test():
|
||||
result = await rate([["西南", 80, 1000], ["华东", 40, 1000]])
|
||||
assert result["verdict"] == {"code": "differs", "subjects": [["西南"], ["华东"]], "direction": "up"}
|
||||
z, p = stats.pooled_z(80, 1000, 40, 1000)
|
||||
assert Fraction(result["values"]["p"]) == pytest.approx(p, rel=1e-9) and result["values"]["test"] == "z"
|
||||
low, high = stats.newcombe(80, 1000, 40, 1000)
|
||||
assert Fraction(result["values"]["ci_lo"]) == pytest.approx(low, rel=1e-9)
|
||||
assert result["values"]["a"] == "0.08" and result["values"]["diff"] == "0.04"
|
||||
assert result["phrase"] == "西南的退款率（8.0%）高于华东（4.0%），差异站得住"
|
||||
lower = await rate([["西南", 80, 1000], ["华东", 40, 1000]], pairs=[["华东", "西南"]])
|
||||
assert lower["verdict"]["direction"] == "down" and lower["phrase"].startswith("华东的退款率（4.0%）低于西南（8.0%）")
|
||||
|
||||
|
||||
async def test_several_groups_each_against_the_rest_with_holm():
|
||||
rows = [["西南", 80, 1000], ["华东", 40, 1000], ["华南", 42, 1000], ["华北", 38, 1000]]
|
||||
result = await rate(rows)
|
||||
table = result["table"]
|
||||
assert [r[0] for r in table["rows"]] == ["西南", "华东", "华南", "华北"]
|
||||
assert table["rows"][0][1] == "其余各地区"
|
||||
raw = [Fraction(r[10]) for r in table["rows"]]
|
||||
adjusted = [Fraction(r[11]) for r in table["rows"]]
|
||||
assert all(a >= p for a, p in zip(adjusted, raw))
|
||||
assert adjusted == pytest.approx(stats.holm([float(p) for p in raw]), rel=1e-9)
|
||||
assert result["verdict"]["code"] == "differs" and result["verdict"]["subjects"] == [["西南"], ["其余各地区"]]
|
||||
assert any(c["code"] == "multiple_comparisons" and c["text"] == "比了 4 对，已按次数校正" for c in result["caveats"])
|
||||
assert result["phrase"] == "比了 4 对，1 对差异站得住：西南的退款率（8.0%）高于其余各地区（4.0%）"
|
||||
ref = await rate(rows, reference="华东")
|
||||
assert [r[0] for r in ref["table"]["rows"]] == ["西南", "华南", "华北"] and ref["params"]["reference"] == "华东"
|
||||
|
||||
|
||||
async def test_rate_call_errors():
|
||||
assert (await call_error("differ", "rate", snaps=[rates([["西南", 5, 4], ["华东", 1, 9]])], query="Q1", by="region",
|
||||
numerator="refunds", denominator="orders")).code == "not_count"
|
||||
assert (await call_error("differ", "rate", snaps=[rates([["西南", 1.5, 4], ["华东", 1, 9]])], query="Q1",
|
||||
by="region", numerator="refunds", denominator="orders")).code == "not_count"
|
||||
unknown = await call_error("differ", "rate", snaps=[rates([["西南", 1, 4], ["华东", 1, 9]])], query="Q1",
|
||||
by="region", numerator="refunds", denominator="orders", pairs=[["西南", "东北"]])
|
||||
assert unknown.code == "unknown_group" and "西南、华东" in unknown.message
|
||||
dup = await call_error("differ", "rate", snaps=[rates([["西南", 1, 4], ["西南", 1, 9]])], query="Q1", by="region",
|
||||
numerator="refunds", denominator="orders")
|
||||
assert dup.code == "duplicate_rows"
|
||||
|
||||
|
||||
async def test_means_from_sums_and_squares_use_welch():
|
||||
sql = "SELECT channel, SUM(amount) AS s, SUM(amount * amount) AS ss, COUNT(*) AS n FROM orders GROUP BY channel"
|
||||
rows = [["线上", 5000, 300000, 100], ["线下", 4000, 200000, 100]]
|
||||
result = await analyze("differ", "mean", snaps=[snap("Q1", sql, ["channel", "s", "ss", "n"], rows)], query="Q1",
|
||||
by="channel", sum="s", sumsq="ss", n="n", label="客单价")
|
||||
va = (300000 - 5000 ** 2 / 100) / 99
|
||||
vb = (200000 - 4000 ** 2 / 100) / 99
|
||||
expected = stats.welch(50, va, 100, 40, vb, 100)
|
||||
assert Fraction(result["values"]["p"]) == pytest.approx(expected.p, rel=1e-9)
|
||||
assert result["verdict"]["code"] == "differs" and result["values"]["test"] == "welch"
|
||||
assert result["phrase"] == "线上的客单价（50）高于线下（40），差异站得住"
|
||||
assert result["method"] == "differ.welch@1" and result["settings"] == {"differ_alpha": 0.05, "min_n": 30}
|
||||
|
||||
|
||||
async def test_means_from_mean_sd_and_n_and_small_samples():
|
||||
sql = "SELECT channel, avg_amount, sd_amount, n FROM channel_stats"
|
||||
rows = [["线上", 52, 12, 12], ["线下", 50, 11, 400]]
|
||||
result = await analyze("differ", "mean", snaps=[snap("Q1", sql, ["channel", "avg_amount", "sd_amount", "n"], rows)],
|
||||
query="Q1", by="channel", mean="avg_amount", sd="sd_amount", n="n", label="客单价")
|
||||
assert result["verdict"]["code"] == "no_evidence"
|
||||
note = next(c for c in result["caveats"] if c["code"] == "small_sample")
|
||||
assert note["text"] == "有一组的样本数不到 30（线上 12），结论不稳"
|
||||
assert result["phrase"].startswith("线上和线下的客单价差异可能只是样本波动（样本量 12、400）")
|
||||
|
||||
|
||||
async def test_rows_mode_flags_a_single_value_that_flips_the_verdict():
|
||||
base = [["线下", v] for v in [40, 60] * 200]
|
||||
online = [["线上", f"{v:.1f}"] for v in [41.3, 61.3] * 200]
|
||||
online[-1] = ["线上", "231.3"]
|
||||
s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], online + base)
|
||||
result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount", label="客单价")
|
||||
assert result["method"] == "differ.welch_rows@1"
|
||||
assert result["verdict"]["code"] == "differs"
|
||||
note = next(c for c in result["caveats"] if c["code"] == "outlier_sensitive")
|
||||
assert note["text"] == "去掉最大的一笔后，差异不再站得住"
|
||||
assert note["text"] in result["phrase"]
|
||||
hidden = [["线上", 50 + (i % 7)] for i in range(30)] + [["线下", 48 + (i % 5)] for i in range(30)] + [["线上", 400]]
|
||||
s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], hidden)
|
||||
flipped = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount", label="客单价")
|
||||
assert flipped["verdict"]["code"] == "no_evidence"
|
||||
assert any(c["text"] == "去掉最大的一笔后，差异变得站得住" for c in flipped["caveats"])
|
||||
|
||||
|
||||
async def test_two_periods_of_the_same_group():
|
||||
sql = "SELECT week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders WHERE region = '西南' GROUP BY week"
|
||||
s = snap("Q1", sql, ["week", "refunds", "orders"], [["W41", 30, 1000], ["W42", 60, 1000], ["W40", 31, 1000]])
|
||||
result = await analyze("differ", "rate", snaps=[s], query="Q1", period="week", base="W41", target="W42",
|
||||
numerator="refunds", denominator="orders", label="退款率")
|
||||
assert result["verdict"] == {"code": "differs", "subjects": [["W42"], ["W41"]], "direction": "up"}
|
||||
assert result["inputs"][0]["rows"] == [0, 1]
|
||||
assert result["phrase"] == "W42 的退款率（6.0%）高于 W41（3.0%），差异站得住"
|
||||
|
||||
|
||||
async def test_a_group_with_one_value_cannot_be_tested():
|
||||
s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], [["线上", 5], ["线下", 4], ["线下", 6]])
|
||||
result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount")
|
||||
assert result["verdict"]["code"] == "no_evidence" and result["values"]["p"] is None
|
||||
assert result["table"]["rows"][0][-1] == "样本不够，算不了"
|
||||
assert math.isfinite(float(result["values"]["diff"]))
|
||||
@@ -0,0 +1,129 @@
|
||||
"""可比性检查的现查（app/analysis/probe.py）：make_prober 在真的数据源上探表的日期范围、在库里算按运行当天现算的界，遮罩列
|
||||
不探、超时限不探、现查不成退回剖析事实；Book 先登记、查完再算，结论只记最后真用到的。库是现造的 SQLite 文件，表名、列名、
|
||||
数字都是虚构的。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import uuid
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from analysis_samples import CATALOGS, recomputed, snap
|
||||
from app import analysis
|
||||
from app.analysis.contract import effective_settings, same
|
||||
from app.analysis.probe import Book, make_prober
|
||||
from app.data import introspect
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
|
||||
CLOCK = "2026-10-21T09:30:00"
|
||||
#: SQLite 的 date('now') 按 UTC 算：表里的数据造到 UTC 的昨天
|
||||
UTC_TODAY = datetime.now(timezone.utc).date()
|
||||
LAST_DAY = UTC_TODAY - timedelta(days=1)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def shop(tmp_path):
|
||||
"""orders：order_date 从 2026-01-01 到昨天（UTC），每天一单；customers 的 signup_date、phone 是遮罩列。"""
|
||||
path = tmp_path / f"{uuid.uuid4().hex[:8]}.db"
|
||||
db = sqlite3.connect(path)
|
||||
db.executescript("CREATE TABLE orders (order_date TEXT, region TEXT, amount INTEGER);"
|
||||
"CREATE TABLE customers (signup_date TEXT, phone TEXT);"
|
||||
"INSERT INTO customers VALUES ('2026-01-05', '13900000000');")
|
||||
day = date(2026, 1, 1)
|
||||
while day <= LAST_DAY:
|
||||
db.execute("INSERT INTO orders VALUES (?, '华东', 100)", (day.isoformat(),))
|
||||
day += timedelta(days=1)
|
||||
db.commit()
|
||||
db.close()
|
||||
name = f"shop_{uuid.uuid4().hex[:8]}"
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True,
|
||||
options={"mask_columns": ["phone", "signup_date"]})
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
row.schema_cache = await introspect.introspect(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
yield row
|
||||
await engines.invalidate(row.id)
|
||||
|
||||
|
||||
async def test_the_live_range_of_a_column(shop):
|
||||
prober = make_prober({shop.name: shop}, clock=lambda: CLOCK)
|
||||
probe = await prober(shop.name, "orders", "order_date")
|
||||
assert probe == {"source": shop.name, "table": "orders", "column": "order_date", "min": "2026-01-01",
|
||||
"max": LAST_DAY.isoformat(), "basis": "live", "at": CLOCK}
|
||||
assert prober.dialect(shop.name) == "sqlite" and prober.dialect("nope") is None
|
||||
assert await prober("nope", "orders", "order_date") is None
|
||||
|
||||
|
||||
async def test_masked_columns_are_never_probed(shop):
|
||||
prober = make_prober({shop.name: shop})
|
||||
assert await prober(shop.name, "customers", "signup_date") is None # 数据源设的遮罩列
|
||||
explicit = make_prober({shop.name: shop}, masked={shop.name: ["ORDER_DATE"]})
|
||||
assert await explicit(shop.name, "orders", "order_date") is None
|
||||
|
||||
|
||||
async def test_a_failed_probe_falls_back_to_profile_facts(shop):
|
||||
async def facts(table):
|
||||
return {"at": "2026-10-20T02:00:00", "date_ranges": [{"column": "paid_on", "min": "2026-02-01",
|
||||
"max": "2026-10-19"}]}
|
||||
|
||||
prober = make_prober({shop.name: shop}, facts={shop.name: facts}, clock=lambda: CLOCK)
|
||||
probe = await prober(shop.name, "payments", "paid_on") # 没有这张表：现查不成
|
||||
assert probe == {"source": shop.name, "table": "payments", "column": "paid_on", "min": "2026-02-01",
|
||||
"max": "2026-10-19", "basis": "profile", "at": "2026-10-20"}
|
||||
|
||||
|
||||
async def test_the_budget_stops_further_probes(shop):
|
||||
prober = make_prober({shop.name: shop}, budget=0)
|
||||
await prober(shop.name, "orders", "order_date")
|
||||
assert await prober(shop.name, "orders", "order_date") is None
|
||||
|
||||
|
||||
async def test_bounds_are_computed_in_the_database(shop):
|
||||
prober = make_prober({shop.name: shop}, clock=lambda: CLOCK)
|
||||
probe = await prober.bounds(shop.name, "DATE('now', '-7 days')")
|
||||
assert probe["expr"] == "DATE('now', '-7 days')" and probe["basis"] == "live" and probe["at"] == CLOCK
|
||||
assert date.fromisoformat(probe["value"]) == UTC_TODAY - timedelta(days=7)
|
||||
assert await prober.bounds(shop.name, "no_such_function(1)") is None
|
||||
|
||||
|
||||
def test_the_book_registers_first_and_records_only_what_was_used():
|
||||
book = Book({})
|
||||
assert book.range("shop", "orders", "order_date") is None and book.bound("shop", "DATE('now')") is None
|
||||
assert book.missing == [("range", "shop", "orders", "order_date"), ("bound", "shop", "DATE('now')")]
|
||||
found = {"source": "shop", "table": "orders", "column": "order_date", "min": "2026-01-01", "max": "2026-10-18",
|
||||
"basis": "live", "at": CLOCK}
|
||||
filled = Book({("range", "shop", "orders", "order_date"): found, ("bound", "shop", "DATE('now')"): None})
|
||||
assert filled.range("shop", "ORDERS", "Order_Date") is found # 表名、列名不分大小写
|
||||
filled.bound("shop", "DATE('now')")
|
||||
assert filled.missing == [] and filled.used == [found]
|
||||
replay = Book([found]) # 复算：记下的就是全部
|
||||
assert replay.bound("shop", "DATE('now', '-1 day')") is None and replay.missing == []
|
||||
|
||||
|
||||
async def test_a_full_comparability_check_against_the_database_records_its_probes(shop):
|
||||
sql = ("SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', '{a}') "
|
||||
"AND order_date < date('now', '{b}')")
|
||||
base = snap("Q1", sql.format(a="-14 days", b="-7 days"), ["gmv"], [[700]], source=shop.name)
|
||||
target = snap("Q2", sql.format(a="-7 days", b="0 days"), ["gmv"], [[300]], source=shop.name)
|
||||
prober = make_prober({shop.name: shop}, clock=lambda: CLOCK)
|
||||
result = await analysis.run({"op": "comparable", "params": {"base_query": "Q1", "target_query": "Q2",
|
||||
"measure": "gmv"}},
|
||||
{"Q1": base, "Q2": target}.get, settings=effective_settings(None),
|
||||
context={"today": UTC_TODAY, "probe": prober,
|
||||
"catalogs": {shop.name: CATALOGS["shop"]}})
|
||||
exprs = [p["expr"] for p in result["probes"] if "expr" in p]
|
||||
assert exprs == ["DATE('now', '-14 days')", "DATE('now', '-7 days')", "DATE('now', '0 days')"]
|
||||
ranges = [p for p in result["probes"] if "table" in p]
|
||||
assert [(p["table"], p["column"], p["max"], p["basis"]) for p in ranges] == \
|
||||
[("orders", "order_date", LAST_DAY.isoformat(), "live")]
|
||||
assert result["values"]["base_days"] == "7" and result["values"]["target_days"] == "7"
|
||||
assert result["values"]["target_end"] == LAST_DAY.isoformat()
|
||||
assert all(i["dialect"] == "sqlite" for i in result["inputs"])
|
||||
assert result["verdict"]["code"] == "comparable" and result["phrase"] == "两期可比"
|
||||
assert same(result, recomputed(result, [base, target]))
|
||||
@@ -0,0 +1,152 @@
|
||||
"""analysis.run / recompute（app/analysis/__init__.py）：结论的形状严格按契约、同输入两次 run 逐字节相同、复算和运行逐字段相同且
|
||||
只用记下的 probes（不现查、不读目录）、调用方写的系统字段不收、复算取不回输入时的说法。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from analysis_samples import CATALOGS, analyze, call_error, fixed_prober, recomputed, snap
|
||||
from app import analysis
|
||||
from app.analysis import contract as c
|
||||
from app.analysis.contract import CallError, effective_settings, same
|
||||
|
||||
WEEKLY_SQL = "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week"
|
||||
WEEKLY = snap("Q2", WEEKLY_SQL, ["region", "week", "gmv"],
|
||||
[["华东", "2026-W41", 820], ["华东", "2026-W42", 700], ["华北", "2026-W41", 600],
|
||||
["华北", "2026-W42", 560], ["华南", "2026-W41", 700], ["华南", "2026-W42", 666]])
|
||||
SERIES = snap("Q1", "SELECT DATE(order_date, 'weekday 0', '-6 days') AS week, SUM(amount) AS gmv FROM orders GROUP BY 1",
|
||||
["week", "gmv"], [[(date(2026, 6, 1) + timedelta(days=7 * i)).isoformat(), 100 + (i % 4) * 3 + i]
|
||||
for i in range(20)])
|
||||
RATES = snap("Q3", "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region",
|
||||
["region", "refunds", "orders"], [["西南", 80, 1000], ["华东", 40, 1000], ["华北", 41, 1000]])
|
||||
BASE = snap("Q4", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' AND "
|
||||
"order_date < '2026-10-12' GROUP BY region", ["region", "gmv"], [["华东", 820], ["华北", 600]])
|
||||
TARGET = snap("Q5", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-12' AND "
|
||||
"order_date < '2026-10-19' GROUP BY region", ["region", "gmv"], [["华东", 700], ["华北", 590]])
|
||||
SNAPS = [WEEKLY, SERIES, RATES, BASE, TARGET]
|
||||
PROBE = {("orders", "order_date"): ("2025-01-01", "2026-10-20")}
|
||||
|
||||
CALLS = [
|
||||
("decompose", "additive", {"query": "Q2", "by": "region", "period": "week", "base": "2026-W41",
|
||||
"target": "2026-W42", "measure": "gmv", "label": "销售额"}),
|
||||
("decompose", "additive", {"base_query": "Q4", "target_query": "Q5", "by": "region", "measure": "gmv"}),
|
||||
("baseline", None, {"query": "Q1", "period": "week", "measure": "gmv"}),
|
||||
("trend", None, {"query": "Q1", "period": "week", "measure": "gmv"}),
|
||||
("differ", "rate", {"query": "Q3", "by": "region", "numerator": "refunds", "denominator": "orders"}),
|
||||
("comparable", None, {"base_query": "Q4", "target_query": "Q5", "measure": "gmv"}),
|
||||
]
|
||||
|
||||
|
||||
async def _run(op, mode, params, *, probe=None, **kw):
|
||||
return await analyze(op, mode, snaps=SNAPS, probe=probe or fixed_prober(PROBE), check=False, **params, **kw)
|
||||
|
||||
|
||||
def _assert_shape(result):
|
||||
"""结论的字段、取值严格按契约。"""
|
||||
assert set(result) == set(c.AnalysisResult.__annotations__)
|
||||
assert result["op"] in c.OPS and result["method"] == c.METHODS[(result["op"], result["mode"])]
|
||||
assert result["status"] in ("ok", "refused")
|
||||
if result["status"] == "refused":
|
||||
assert result["reason_code"] in c.REASONS and result["reason"] and result["verdict"] is None
|
||||
else:
|
||||
assert result["reason_code"] is None and result["reason"] is None
|
||||
assert result["verdict"]["code"] in c.VERDICTS[result["op"]]
|
||||
assert set(result["verdict"]) == {"code", "subjects", "direction"}
|
||||
assert all(isinstance(s, list) and all(isinstance(x, str) for x in s) for s in result["verdict"]["subjects"])
|
||||
assert result["verdict"]["direction"] in (None, "up", "down")
|
||||
table = result["table"]
|
||||
assert set(table) == {"columns", "labels", "rows"} and set(table["labels"]) == set(table["columns"])
|
||||
assert all(len(r) == len(table["columns"]) for r in table["rows"])
|
||||
assert all(v is None or isinstance(v, str) for v in result["values"].values())
|
||||
assert all(set(cv) == {"code", "text"} and cv["code"] in c.CAVEATS for cv in result["caveats"])
|
||||
assert set(result["settings"]) <= set(c.DEFAULTS)
|
||||
assert all(i.get("artifact") for i in result["inputs"])
|
||||
assert result["criterion"] and result["phrase"]
|
||||
json.dumps(result, ensure_ascii=False) # 写得进 JSON
|
||||
|
||||
|
||||
@pytest.mark.parametrize("op,mode,params", CALLS)
|
||||
async def test_every_result_has_the_contract_shape_and_recomputes_the_same(op, mode, params):
|
||||
result = await _run(op, mode, params)
|
||||
_assert_shape(result)
|
||||
again = recomputed(result, SNAPS)
|
||||
assert same(result, again)
|
||||
assert json.dumps(again, sort_keys=True, ensure_ascii=False) == json.dumps(result, sort_keys=True, ensure_ascii=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("op,mode,params", CALLS)
|
||||
async def test_the_same_inputs_give_byte_identical_results(op, mode, params):
|
||||
one = await _run(op, mode, params)
|
||||
two = await _run(op, mode, params)
|
||||
assert json.dumps(one, sort_keys=True, ensure_ascii=False) == json.dumps(two, sort_keys=True, ensure_ascii=False)
|
||||
|
||||
|
||||
async def test_recompute_uses_the_recorded_probes_and_facts_not_live_ones():
|
||||
result = await _run("comparable", None, {"base_query": "Q4", "target_query": "Q5", "measure": "gmv"})
|
||||
assert result["verdict"]["code"] == "comparable" and len(result["probes"]) == 1
|
||||
# 记下的范围换一个（表里数据只到 10-16）：复算按记下的算，结论跟着变——它没有现查
|
||||
edited = {**result, "probes": [{**result["probes"][0], "max": "2026-10-16"}]}
|
||||
assert recomputed(edited, SNAPS)["verdict"]["code"] == "incomparable"
|
||||
# 目录事后改了也不影响复算：目录事实记在 params 里
|
||||
decompose = await _run(*CALLS[0])
|
||||
assert decompose["params"]["catalog"]["labels"]["region"] == "地区"
|
||||
CATALOGS["shop"]["orders"]["columns"]["region"]["label"]["value"] = "大区"
|
||||
try:
|
||||
assert same(decompose, recomputed(decompose, SNAPS))
|
||||
finally:
|
||||
CATALOGS["shop"]["orders"]["columns"]["region"]["label"]["value"] = "地区"
|
||||
|
||||
|
||||
async def test_system_params_written_by_the_caller_are_ignored():
|
||||
params = {"query": "Q1", "period": "week", "measure": "gmv", "today": "2020-01-01",
|
||||
"catalog": {"labels": {"gmv": "假的叫法"}}, "depth": 9}
|
||||
result = await analysis.run({"op": "baseline", "params": params}, {s["ref"]: s for s in SNAPS}.get,
|
||||
settings=effective_settings(None),
|
||||
context={"today": date(2026, 10, 21), "catalogs": CATALOGS})
|
||||
assert result["params"]["today"] == "2026-10-21"
|
||||
assert "假的叫法" not in result["phrase"] and "depth" not in result["params"]
|
||||
|
||||
|
||||
async def test_analysis_card_overrides_reach_the_criterion_and_settings():
|
||||
params = CALLS[0][2]
|
||||
result = await _run("decompose", "additive", params, settings={"main_share": 0.6})
|
||||
assert result["settings"]["main_share"] == 0.6 and "占总变化的 60% 以上" in result["criterion"]
|
||||
|
||||
|
||||
async def test_call_errors_for_unknown_ops_and_modes():
|
||||
with pytest.raises(CallError) as e:
|
||||
await analysis.run({"op": "forecast", "params": {}}, {}.get, settings=effective_settings(None), context={})
|
||||
assert e.value.code == "unknown_op" and "变化拆解" in e.value.message
|
||||
with pytest.raises(CallError) as e:
|
||||
await analysis.run({"op": "decompose", "mode": "share", "params": {}}, {}.get,
|
||||
settings=effective_settings(None), context={})
|
||||
assert e.value.code == "unknown_mode" and "additive / ratio / factor" in e.value.message
|
||||
missing = await call_error("baseline", snaps=SNAPS, query="Q1", period="week")
|
||||
assert missing.code == "missing_param" and missing.message == "没写度量列"
|
||||
|
||||
|
||||
async def test_recompute_says_when_an_input_cannot_be_loaded():
|
||||
result = await _run(*CALLS[0])
|
||||
with pytest.raises(CallError) as e:
|
||||
analysis.recompute(result, lambda artifact: None)
|
||||
assert e.value.code == "input_missing" and e.value.message == "复算取不回 Q2 的查询快照"
|
||||
|
||||
|
||||
async def test_the_hypothesis_is_recorded_but_never_rendered():
|
||||
result = await _run(*CALLS[0], hypothesis="可能是华东促销结束 30% 的影响")
|
||||
assert result["hypothesis"] == "可能是华东促销结束 30% 的影响"
|
||||
assert "促销" not in result["phrase"] and "30" not in result["phrase"]
|
||||
assert same(result, {**recomputed(result, SNAPS), "hypothesis": None})
|
||||
|
||||
|
||||
async def test_ledger_entries_name_the_real_input_snapshots():
|
||||
result = await _run(*CALLS[1])
|
||||
entry = c.ledger_entry(result, node_id="agent", exec_no=1, artifact="f" * 64, call_id="call-1")
|
||||
assert entry["inputs"] == [BASE["artifact"], TARGET["artifact"]] and entry["op"] == "decompose"
|
||||
|
||||
|
||||
def test_the_call_limit_error_is_spoken_with_the_limit():
|
||||
error = analysis.exhausted({"max_calls": 5})
|
||||
assert error.code == "max_calls" and error.message == "这次运行的分析次数已经用完（最多 5 次）：用已有的分析结论写，不再新做分析"
|
||||
@@ -0,0 +1,210 @@
|
||||
"""波动基线、趋势（app/analysis/series.py，细案 01 §3、§4）：常规 / 偏离 / 明显偏离、σ̂ = 0、历史不够、本期不完整、缺期、日数据
|
||||
lag 7、新高新低；上升 / 下降 / 无趋势、连续期数、缺期。每个用例都顺带核对复算和运行逐字段相同。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, timedelta
|
||||
from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
|
||||
from analysis_samples import analyze, call_error, fixed_prober, snap
|
||||
from app.analysis.series import MAD_SCALE, run_length
|
||||
|
||||
SQL = "SELECT DATE(order_date, 'weekday 0', '-6 days') AS week, COUNT(*) AS orders FROM orders GROUP BY 1"
|
||||
#: 每周一的日期：最后一周是 2026-10-12（到 10-18，运行当天 10-21 之前已经过完）
|
||||
LAST_MONDAY = date(2026, 10, 12)
|
||||
|
||||
|
||||
def weeks(values, *, last=LAST_MONDAY):
|
||||
start = last - timedelta(days=7 * (len(values) - 1))
|
||||
return [[(start + timedelta(days=7 * i)).isoformat(), v] for i, v in enumerate(values)]
|
||||
|
||||
|
||||
def series(rows, sql=SQL, cols=("week", "orders")):
|
||||
return snap("Q1", sql, list(cols), rows, types={cols[0]: "text", cols[1]: "number"})
|
||||
|
||||
|
||||
async def baseline(rows, **params):
|
||||
params = {"query": "Q1", "period": "week", "measure": "orders", "label": "订单数", **params}
|
||||
return await analyze("baseline", snaps=[series(rows)], **params)
|
||||
|
||||
|
||||
async def trend(rows, **params):
|
||||
params = {"query": "Q1", "period": "week", "measure": "orders", "label": "线上销售额", **params}
|
||||
return await analyze("trend", snaps=[series(rows)], **params)
|
||||
|
||||
|
||||
NOISE = [100, 102, 98, 101, 99, 103, 100, 97, 102, 101, 99, 100, 98, 103, 101, 100, 99, 102, 100]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 波动基线
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_level_basis_with_hand_computed_median_and_mad():
|
||||
rows = [[i + 1, v] for i, v in enumerate([10, 12, 11, 13, 9, 10, 12, 11, 16])]
|
||||
result = await baseline(rows, basis="level")
|
||||
# 历史中位数 11，MAD 1，σ̂ = 1.4826，z = (16 − 11) / 1.4826
|
||||
values = result["values"]
|
||||
assert values["median"] == "11" and values["n_history"] == "8" and values["rank"] == "1"
|
||||
assert Fraction(values["z"]) == pytest.approx(Fraction(5) / MAD_SCALE)
|
||||
assert values["band_lo"] == "8.0348" and values["band_hi"] == "13.9652"
|
||||
assert result["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "up"}
|
||||
assert result["phrase"].startswith("订单数本期为 16，明显超出过去 8 期的常规波动范围（8.03 至 13.97）")
|
||||
assert any(c["code"] == "record_high" for c in result["caveats"])
|
||||
assert result["params"]["grain"] == "seq" and result["params"]["lag"] == 1
|
||||
assert result["settings"] == {"k_warn": 2.0, "k_alert": 3.0, "min_history": 8, "max_history": 52}
|
||||
assert result["table"]["rows"][-1][-1] == "本期"
|
||||
|
||||
|
||||
async def test_change_basis_within_outside_and_far_outside():
|
||||
probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")})
|
||||
within = await analyze("baseline", snaps=[series(weeks(NOISE + [101]))], probe=probe, query="Q1", period="week",
|
||||
measure="orders", label="订单数")
|
||||
assert within["verdict"]["code"] == "within_band"
|
||||
assert within["phrase"].startswith("订单数本期变化 +1.0%，在过去 18 周的常规波动范围内（")
|
||||
assert within["params"]["today"] == "2026-10-21" and within["params"]["grain"] == "week"
|
||||
assert within["probes"][0]["table"] == "orders" and within["probes"][0]["column"] == "order_date"
|
||||
assert not any(c["code"] == "comparability" for c in within["caveats"])
|
||||
outside = await analyze("baseline", snaps=[series(weeks(NOISE + [107]))], probe=probe, query="Q1",
|
||||
period="week", measure="orders")
|
||||
assert outside["verdict"]["code"] == "outside_band" and "超出过去 18 周" in outside["phrase"]
|
||||
far = await analyze("baseline", snaps=[series(weeks(NOISE + [70]))], probe=probe, query="Q1", period="week",
|
||||
measure="orders", label="订单数")
|
||||
assert far["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "down"}
|
||||
assert far["phrase"].startswith("订单数本期变化 -30.0%，明显超出过去 18 周的常规波动范围")
|
||||
assert any(c["code"] == "record_low" and c["text"] == "本期的订单数是过去 18 周里最低的" for c in far["caveats"])
|
||||
|
||||
|
||||
async def test_without_a_table_range_completeness_is_said_to_be_unverified_but_not_in_the_phrase():
|
||||
result = await baseline(weeks(NOISE + [101]))
|
||||
note = next(c for c in result["caveats"] if c["code"] == "comparability")
|
||||
assert note["text"] == "没能核对本期的数据是否到齐（查不到表里这一列的日期范围）"
|
||||
assert note["text"] not in result["phrase"]
|
||||
|
||||
|
||||
async def test_flat_history_has_no_spread():
|
||||
rows = [[i + 1, 50] for i in range(10)]
|
||||
same = await baseline(rows, basis="level")
|
||||
assert same["verdict"]["code"] == "within_band" and same["values"]["z"] is None
|
||||
assert same["phrase"] == "订单数本期为 50，和过去 9 期一样（每期都是 50）"
|
||||
jump = await baseline(rows[:-1] + [[10, 51]], basis="level")
|
||||
assert jump["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "up"}
|
||||
assert jump["phrase"].startswith("订单数本期为 51，明显超出过去 9 期的常规波动范围（每期都是 50）")
|
||||
|
||||
|
||||
async def test_short_history_partial_target_gaps_and_zero_bases_are_refused():
|
||||
short = await baseline(weeks([100, 101, 99, 102, 98, 100]))
|
||||
# 6 周：第一周没有上一周可比，本期不算历史，历史的变化只有 4 个
|
||||
assert short["reason_code"] == "history_short" and short["reason"] == "只有 4 期历史，至少要 8 期"
|
||||
assert short["phrase"] == "波动基线没有做成：只有 4 期历史，至少要 8 期"
|
||||
# 最后一周是 10-19 到 10-25：运行当天 10-21 还没过完
|
||||
partial = await baseline(weeks(NOISE + [60], last=date(2026, 10, 19)))
|
||||
assert partial["reason_code"] == "partial_target"
|
||||
assert partial["reason"] == "本期数据只到 2026-10-20，还差 5 天，和完整的周比没有意义"
|
||||
# 周过完了，但表里的数据只到 10-17
|
||||
probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-17")})
|
||||
late = await analyze("baseline", snaps=[series(weeks(NOISE + [60]))], probe=probe, query="Q1", period="week",
|
||||
measure="orders")
|
||||
assert late["reason"] == "本期数据只到 2026-10-17，还差 1 天，和完整的周比没有意义"
|
||||
gap_rows = weeks(NOISE + [100, 100])
|
||||
del gap_rows[-2] # 本期的上一周没有
|
||||
gap = await baseline(gap_rows)
|
||||
assert gap["reason_code"] == "target_gap"
|
||||
zero = await baseline(weeks(NOISE + [0, 10]))
|
||||
assert zero["reason_code"] == "zero_base"
|
||||
unreadable = await baseline([["上周", 1], ["本周", 2]])
|
||||
assert unreadable["reason_code"] == "period_unreadable" and "上周" in unreadable["reason"]
|
||||
|
||||
|
||||
async def test_daily_series_compare_with_the_same_weekday_last_week():
|
||||
start = date(2026, 7, 1)
|
||||
rows = [[(start + timedelta(days=i)).isoformat(), 100 + (i % 7) * 10] for i in range(100)]
|
||||
result = await analyze("baseline", snaps=[series(rows, cols=("day", "orders"))], query="Q1", period="day",
|
||||
measure="orders", label="订单数", check=True)
|
||||
assert result["params"]["grain"] == "day" and result["params"]["lag"] == 7
|
||||
# 星期效应：和上周同一天比，每天的变化都是 0
|
||||
assert result["values"]["median"] == "0" and result["verdict"]["code"] == "within_band"
|
||||
assert result["phrase"].startswith("订单数比上周同一天变化 0.0%，和过去 52 天一样")
|
||||
yoy = await baseline(weeks(NOISE * 3), lag=52) # 57 周：能和 52 周前比的历史只有 4 期
|
||||
assert yoy["reason_code"] == "history_short" and yoy["reason"] == "只有 4 期历史，至少要 8 期"
|
||||
assert (await baseline(weeks(NOISE * 4), lag=52))["status"] == "ok"
|
||||
assert (await call_error("baseline", snaps=[series(weeks(NOISE))], query="Q1", period="week", measure="orders",
|
||||
lag=0)).code == "bad_lag"
|
||||
|
||||
|
||||
async def test_duplicate_periods_and_given_targets():
|
||||
rows = weeks(NOISE + [101])
|
||||
error = await call_error("baseline", snaps=[series(rows + [rows[-1]])], query="Q1", period="week",
|
||||
measure="orders")
|
||||
assert error.code == "duplicate_rows" and "在 SQL 里按 week 汇总" in error.message
|
||||
earlier = await baseline(rows, target=rows[-3][0])
|
||||
assert earlier["params"]["target"] == rows[-3][0] and earlier["values"]["n_history"] == "16"
|
||||
missing = await call_error("baseline", snaps=[series(rows)], query="Q1", period="week", measure="orders",
|
||||
target="2020-01-06")
|
||||
assert missing.code == "period_not_found"
|
||||
|
||||
|
||||
async def test_iso_week_labels_and_months():
|
||||
labels = [[f"2026-W{w:02d}", v] for w, v in zip(range(22, 42), NOISE + [101])]
|
||||
result = await baseline(labels)
|
||||
assert result["params"]["grain"] == "week" and result["verdict"]["code"] == "within_band"
|
||||
months = [[f"2025-{m:02d}", 100 + m] for m in range(1, 13)] + [[f"2026-{m:02d}", 112 + m] for m in range(1, 10)]
|
||||
monthly = await baseline(months)
|
||||
assert monthly["params"]["grain"] == "month" and "过去 19 个月" in monthly["phrase"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 趋势
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_rising_and_falling_trends_with_run_length():
|
||||
rising = await trend(weeks([100 + 3 * i + (i % 3) for i in range(12)]))
|
||||
assert rising["verdict"] == {"code": "rising", "subjects": [], "direction": "up"}
|
||||
assert rising["values"]["n"] == "12" and rising["values"]["run_length"] == "11"
|
||||
assert rising["phrase"].startswith("过去 12 周线上销售额呈上升趋势（每周约 +")
|
||||
assert "，已连续 11 周上升" in rising["phrase"]
|
||||
assert rising["settings"] == {"trend_min_points": 6, "trend_alpha": 0.05, "max_history": 52}
|
||||
falling = await trend(weeks([200, 196, 197, 190, 185, 186, 180, 181, 175, 170]))
|
||||
assert falling["verdict"]["code"] == "falling"
|
||||
assert falling["values"]["run_length"] == "2" and "已连续" not in falling["phrase"]
|
||||
|
||||
|
||||
async def test_no_trend_and_exact_small_sample_p():
|
||||
flat = await trend(weeks([100, 104, 98, 103, 97, 101, 99, 102]))
|
||||
assert flat["verdict"] == {"code": "no_trend", "subjects": [], "direction": None}
|
||||
assert flat["phrase"] == "过去 8 周线上销售额没有明显的上升或下降趋势"
|
||||
six = await trend(weeks([1, 2, 3, 4, 5, 6]))
|
||||
# 没有并列、n ≤ 10：精确分布，单调的 6 期双侧 p = 2 / 6!
|
||||
assert Fraction(six["values"]["p"]) == pytest.approx(Fraction(2, 720))
|
||||
assert six["verdict"]["code"] == "rising" and six["values"]["slope"] == "1"
|
||||
|
||||
|
||||
def test_run_length_counts_consecutive_changes_from_the_end():
|
||||
assert run_length([Fraction(v) for v in (1, 2, 3, 2, 3, 4, 5)]) == (3, "up")
|
||||
assert run_length([Fraction(v) for v in (5, 4, 3)]) == (2, "down")
|
||||
assert run_length([Fraction(v) for v in (1, 2, 2)]) == (0, None)
|
||||
|
||||
|
||||
async def test_trend_needs_enough_consecutive_points():
|
||||
rows = weeks([100 + i for i in range(10)])
|
||||
del rows[-4] # 最后连着的只有 3 期
|
||||
gap = await trend(rows)
|
||||
assert gap["reason_code"] == "series_gap" and gap["reason"] == "最后连着的只有 3 期，再往前缺期（至少要连着的 6 期）"
|
||||
short = await trend(weeks([1, 2, 3, 4]))
|
||||
assert short["reason_code"] == "history_short" and short["reason"] == "只有 4 期，至少要 6 期"
|
||||
partial = await trend(weeks([100 + i for i in range(10)], last=date(2026, 10, 19)))
|
||||
assert partial["reason_code"] == "partial_target"
|
||||
|
||||
|
||||
async def test_daily_trends_warn_about_weekday_effects():
|
||||
start = date(2026, 8, 1)
|
||||
rows = [[(start + timedelta(days=i)).isoformat(), 100 + i] for i in range(30)]
|
||||
result = await analyze("trend", snaps=[series(rows, cols=("day", "orders"))], query="Q1", period="day",
|
||||
measure="orders")
|
||||
assert result["verdict"]["code"] == "rising"
|
||||
assert any(c["code"] == "weekday_effect" and c["text"] == "日数据有星期效应，看趋势建议按周汇总"
|
||||
for c in result["caveats"])
|
||||
assert "日数据有星期效应" in result["phrase"]
|
||||
Reference in New Issue
Block a user