mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 01:21:59 +00:00
Merge branch 'an/w1' into an1/d
This commit is contained in:
@@ -8,14 +8,24 @@
|
||||
- recompute:按结论里记下的 params、settings、probes 和输入快照(按 sha256 取回)重算一遍,不现查;和封存的用
|
||||
contract.same 比。
|
||||
|
||||
契约(名字、形状、缺省阈值)在 contract.py。
|
||||
契约(名字、形状、缺省阈值)在 contract.py。各算子在 decompose.py(变化拆解)、series.py(波动基线、趋势)、differ.py
|
||||
(差异检验)、comparable.py(可比性检查);统计在 stats.py,判定语和数的写法在 phrases.py,现查在 probe.py。
|
||||
|
||||
**一份结论只由 inputs(快照里的格)、probes、params、settings、method 决定**:run 时从 context 读到的东西(运行当天、目录
|
||||
里的度量类型和叫法、现查的范围)都记进结论——运行当天、目录事实在 params(today、catalog),现查结果在 probes——复算只读
|
||||
记下的。同样的输入两次 run 逐字节相同(现查结果相同时);recompute 和 run 逐字段相同。
|
||||
|
||||
**参数**(AnalysisRequest.params;列名不分大小写,按快照认)见各算子文件头;params 里的 catalog、today、depth 由系统填,
|
||||
调用方写了也不收。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
from app.analysis.contract import AnalysisRequest, AnalysisResult, Context, InputSnapshot
|
||||
from app.analysis.contract import (MODES, OPS, AnalysisRequest, AnalysisResult, CallError, Context, InputSnapshot,
|
||||
effective_settings)
|
||||
|
||||
#: 按调用里写的名字取回输入快照;没有这个名字返回 None
|
||||
Resolver = Callable[[str], InputSnapshot | None]
|
||||
@@ -23,16 +33,145 @@ Resolver = Callable[[str], InputSnapshot | None]
|
||||
ParentResolver = Callable[[str], AnalysisResult | None]
|
||||
|
||||
|
||||
def _op_mode(request: Mapping[str, Any]) -> tuple[str, str | None]:
|
||||
from app.analysis.phrases import OP_NAMES
|
||||
|
||||
op = request.get("op")
|
||||
if op not in OPS:
|
||||
raise CallError("unknown_op", f"不认识的分析:{op}。能做的有{'、'.join(OP_NAMES[o] for o in OPS)}")
|
||||
modes = MODES.get(op)
|
||||
mode = request.get("mode")
|
||||
if modes is None:
|
||||
return op, None
|
||||
if mode not in modes:
|
||||
raise CallError("unknown_mode", f"{OP_NAMES[op]}要写模式:{' / '.join(modes)}")
|
||||
return op, mode
|
||||
|
||||
|
||||
def _compute(op: str, mode: str | None, raw: Mapping[str, Any], env: Any) -> AnalysisResult:
|
||||
from app.analysis import comparable, decompose, differ, series
|
||||
|
||||
if op == "decompose":
|
||||
return {"additive": decompose.additive, "ratio": decompose.ratio, "factor": decompose.factor}[mode](raw, env) # type: ignore[index]
|
||||
if op == "baseline":
|
||||
return series.baseline(raw, env)
|
||||
if op == "trend":
|
||||
return series.trend(raw, env)
|
||||
if op == "differ":
|
||||
return differ.differ(raw, env)
|
||||
return comparable.comparable(raw, env)
|
||||
|
||||
|
||||
def _env(op: str, mode: str | None, settings: Mapping[str, Any], inputs: Any, facts: Any, book: Any,
|
||||
today: date | None, parents: ParentResolver | None, hypothesis: Any) -> Any:
|
||||
from app.analysis.common import Env
|
||||
|
||||
return Env(op=op, mode=mode, settings=settings, inputs=inputs, facts=facts, book=book, today=today,
|
||||
parents=parents, hypothesis=hypothesis if isinstance(hypothesis, str) else None)
|
||||
|
||||
|
||||
async def run(request: AnalysisRequest, resolve: Resolver, *, settings: Mapping[str, Any], context: Context,
|
||||
parents: ParentResolver | None = None) -> AnalysisResult:
|
||||
"""算一份分析结论。settings 是 contract.effective_settings 算好的阈值。"""
|
||||
raise NotImplementedError("analysis.run:第 1 波 W1-a 实现")
|
||||
from app.analysis.common import DERIVED_PARAMS
|
||||
from app.analysis.facts import Facts
|
||||
from app.analysis.inputs import Inputs
|
||||
from app.analysis.probe import MAX_ROUNDS, Book, fetch
|
||||
|
||||
op, mode = _op_mode(request)
|
||||
raw = {k: v for k, v in (request.get("params") or {}).items() if k not in DERIVED_PARAMS}
|
||||
full = effective_settings(settings)
|
||||
prober = context.get("probe")
|
||||
today = context.get("today") or date.today()
|
||||
if isinstance(today, datetime):
|
||||
today = today.date()
|
||||
|
||||
def dialect(name: str, snap: Mapping[str, Any]) -> str | None:
|
||||
given = snap.get("dialect")
|
||||
if isinstance(given, str) and given:
|
||||
return given
|
||||
lookup = getattr(prober, "dialect", None)
|
||||
return lookup(str(snap.get("source") or "")) if lookup is not None else None
|
||||
|
||||
known: dict[tuple[str, ...], Any] = {}
|
||||
for rounds in range(MAX_ROUNDS + 1):
|
||||
book = Book(known)
|
||||
env = _env(op, mode, full, Inputs(resolve, dialect), Facts(catalogs=context.get("catalogs")), book, today,
|
||||
parents, request.get("hypothesis"))
|
||||
result = _compute(op, mode, raw, env)
|
||||
missing = book.missing
|
||||
if not missing or rounds == MAX_ROUNDS:
|
||||
return result
|
||||
known.update(await fetch(book, context))
|
||||
return result # pragma: no cover - 循环里一定返回
|
||||
|
||||
|
||||
def recompute(result: AnalysisResult, load: Callable[[str], InputSnapshot | None], *,
|
||||
load_result: Callable[[str], AnalysisResult | None] | None = None) -> AnalysisResult:
|
||||
"""复算:load 按 sha256 取回输入快照,load_result 按 sha256 取回 parent 那份结论。"""
|
||||
raise NotImplementedError("analysis.recompute:第 1 波 W1-a 实现")
|
||||
"""复算:load 按 sha256 取回输入快照,load_result 按 sha256 取回 parent 那份结论。用结论里记下的 params(含运行当天、
|
||||
目录事实)、settings、probes,不现查、不读目录。取不回输入时抛 CallError(code input_missing);方法升版后调用不再成立
|
||||
时同样抛 CallError(照 run 的说法)。和封存的那份用 contract.same 比。"""
|
||||
from app.analysis.common import DERIVED_PARAMS
|
||||
from app.analysis.facts import Facts
|
||||
from app.analysis.inputs import _CELL, Inputs
|
||||
from app.analysis.probe import Book
|
||||
|
||||
op, mode = _op_mode(result)
|
||||
params = dict(result.get("params") or {})
|
||||
settings = effective_settings(result.get("settings"))
|
||||
names: dict[str, tuple[str | None, str | None, list[str]]] = {}
|
||||
parents_at: dict[str, str] = {}
|
||||
for item in result.get("inputs") or []:
|
||||
ref = str(item.get("ref") or "")
|
||||
if item.get("role") == "parent":
|
||||
parents_at[ref] = str(item.get("artifact") or "")
|
||||
continue
|
||||
cell = _CELL.match(ref) if "row" in item else None
|
||||
name = cell.group("name") if cell else ref
|
||||
names.setdefault(name, (item.get("artifact"), item.get("dialect"), list(item.get("masked") or [])))
|
||||
|
||||
def resolve(name: str) -> InputSnapshot | None:
|
||||
artifact, _, masked = names.get(name, (None, None, []))
|
||||
found = load(artifact) if artifact else None
|
||||
if not isinstance(found, Mapping):
|
||||
raise CallError("input_missing", f"复算取不回 {name} 的查询快照")
|
||||
return {**found, "artifact": artifact, "masked": masked} # type: ignore[typeddict-item]
|
||||
|
||||
def dialect(name: str, snap: Mapping[str, Any]) -> str | None:
|
||||
return names.get(name, (None, None, []))[1]
|
||||
|
||||
def parents(name: str) -> AnalysisResult | None:
|
||||
artifact = parents_at.get(name)
|
||||
found = load_result(artifact) if artifact and load_result is not None else None
|
||||
if not isinstance(found, Mapping):
|
||||
raise CallError("input_missing", f"复算取不回上一层的分析结论 {name}")
|
||||
return found # type: ignore[return-value]
|
||||
|
||||
today = date.fromisoformat(params["today"]) if isinstance(params.get("today"), str) else None
|
||||
env = _env(op, mode, settings, Inputs(resolve, dialect), Facts(recorded=params.get("catalog") or {}),
|
||||
Book(result.get("probes") or []), today, parents, result.get("hypothesis"))
|
||||
raw = {k: v for k, v in params.items() if k not in DERIVED_PARAMS}
|
||||
return _compute(op, mode, raw, env)
|
||||
|
||||
|
||||
__all__ = ["ParentResolver", "Resolver", "recompute", "run"]
|
||||
def exhausted(settings: Mapping[str, Any]) -> CallError:
|
||||
"""本运行分析次数用完(max_calls)的调用错误:入口数着次数,到了上限交回它,不再算。"""
|
||||
limit = int(effective_settings(settings)["max_calls"])
|
||||
return CallError("max_calls", f"这次运行的分析次数已经用完(最多 {limit} 次):用已有的分析结论写,不再新做分析")
|
||||
|
||||
|
||||
def make_prober(*args: Any, **kwargs: Any) -> Any:
|
||||
"""见 app/analysis/probe.make_prober。"""
|
||||
from app.analysis.probe import make_prober as make
|
||||
|
||||
return make(*args, **kwargs)
|
||||
|
||||
|
||||
def comparable_reasons(result: Mapping[str, Any]) -> list[str]:
|
||||
"""可比性结论里 fail、warn 的原因,一句一个(口径卡指标的 comparability.reasons)。"""
|
||||
from app.analysis.comparable import reasons
|
||||
|
||||
return reasons(result)
|
||||
|
||||
|
||||
__all__ = ["ParentResolver", "Resolver", "comparable_reasons", "exhausted", "make_prober", "recompute", "run"]
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
"""算子共用的:一次计算的环境(输入、目录事实、现查簿、阈值)、结论的组装、几样参数的规范化。
|
||||
|
||||
每个算子是 (raw params, Env) → AnalysisResult 的纯函数:只经 Env 取输入快照(Inputs)、目录事实(Facts)、现查结果
|
||||
(Book),阈值只来自 Env.settings。run 和 recompute 只是装 Env 的方式不同(见 app/analysis/__init__.py)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis.contract import METHODS, AnalysisResult, CallError
|
||||
from app.analysis.facts import Facts
|
||||
from app.analysis.inputs import Inputs
|
||||
from app.analysis.numbers import to_fraction
|
||||
from app.analysis.probe import Book
|
||||
|
||||
#: 结论的 params 里由系统填、不收调用方的几项:目录事实、运行当天、下钻的层数
|
||||
DERIVED_PARAMS = frozenset({"catalog", "today", "depth"})
|
||||
|
||||
|
||||
@dataclass
|
||||
class Env:
|
||||
op: str
|
||||
mode: str | None
|
||||
settings: Mapping[str, Any]
|
||||
inputs: Inputs
|
||||
facts: Facts
|
||||
book: Book
|
||||
today: date | None
|
||||
parents: Callable[[str], Mapping[str, Any] | None] | None
|
||||
hypothesis: str | None
|
||||
|
||||
def threshold(self, key: str) -> Fraction:
|
||||
value = to_fraction(self.settings[key])
|
||||
assert value is not None, key
|
||||
return value
|
||||
|
||||
|
||||
def label_for(env: Env, raw: Any, snap: Any, column: str | None, *, default: str | None = None) -> str:
|
||||
"""判定语里的度量叫法:agent 给的 label → 目录里的中文名 → default → 列名,依次取第一个消毒后不为空的(agent 给了能用的
|
||||
就不去查目录)。"""
|
||||
if ph.clean_label(raw):
|
||||
return ph.clean_label(raw)
|
||||
catalog = env.facts.label(snap, column) if column else None
|
||||
return ph.pick_label(catalog, default, column)
|
||||
|
||||
|
||||
def column_label(env: Env, snap: Any, column: str) -> str:
|
||||
"""维度、因子这类列的叫法:目录里的中文名,没有就是列名(都消毒)。"""
|
||||
return ph.pick_label(env.facts.label(snap, column), column)
|
||||
|
||||
|
||||
def today_param(env: Env) -> str:
|
||||
"""要判本期完不完整的算子把运行当天记进 params(复算用记下的那一天)。"""
|
||||
day = env.today or date.today()
|
||||
env.today = day
|
||||
return day.isoformat()
|
||||
|
||||
|
||||
def build(env: Env, params: dict[str, Any], used: Sequence[str], *, status: str = "ok",
|
||||
reason_code: str | None = None, reason: str | None = None, table: Mapping[str, Any] | None = None,
|
||||
values: Mapping[str, Any] | None = None, verdict: Mapping[str, Any] | None = None, criterion: str,
|
||||
phrase: str, caveats: Sequence[Mapping[str, str]] = ()) -> AnalysisResult:
|
||||
"""组装一份结论:形状严格按 contract.AnalysisResult,params 补上查过的目录事实。"""
|
||||
record = env.facts.record()
|
||||
out_params = {k: v for k, v in params.items() if v is not None}
|
||||
if record:
|
||||
out_params["catalog"] = record
|
||||
return {
|
||||
"op": env.op,
|
||||
"mode": env.mode,
|
||||
"method": METHODS[(env.op, env.mode)],
|
||||
"params": out_params,
|
||||
"settings": {k: env.settings[k] for k in used},
|
||||
"inputs": list(env.inputs.records),
|
||||
"probes": env.book.used,
|
||||
"status": status, # type: ignore[typeddict-item]
|
||||
"reason_code": reason_code,
|
||||
"reason": reason,
|
||||
"table": dict(table) if table is not None else None, # type: ignore[typeddict-item]
|
||||
"values": dict(values or {}),
|
||||
"verdict": dict(verdict) if verdict is not None else None, # type: ignore[typeddict-item]
|
||||
"criterion": criterion,
|
||||
"phrase": phrase,
|
||||
"caveats": [dict(c) for c in caveats], # type: ignore[misc]
|
||||
"hypothesis": env.hypothesis,
|
||||
}
|
||||
|
||||
|
||||
def refuse(env: Env, params: dict[str, Any], used: Sequence[str], code: str, reason: str, *,
|
||||
criterion: str, caveats: Sequence[Mapping[str, str]] = ()) -> AnalysisResult:
|
||||
"""算不了(status refused):判定语是「{算子的叫法}没有做成:{原因}」。"""
|
||||
return build(env, params, used, status="refused", reason_code=code, reason=reason, criterion=criterion,
|
||||
phrase=ph.refused(env.op, reason), caveats=caveats)
|
||||
|
||||
|
||||
def need(raw: Mapping[str, Any], key: str, what: str) -> Any:
|
||||
"""必填的参数;没写是调用错误。"""
|
||||
value = raw.get(key)
|
||||
if value is None or (isinstance(value, str) and not value.strip()) or value == []:
|
||||
raise CallError("missing_param", f"没写{what}")
|
||||
return value
|
||||
|
||||
|
||||
def written_places(raw: Any) -> int:
|
||||
"""快照里一格写到第几位小数(headline 是取整过的,按它的书写精度容差):数字文本按字面,浮点按最短写法,整数为 0。"""
|
||||
if isinstance(raw, bool) or raw is None:
|
||||
return 0
|
||||
text = repr(raw) if isinstance(raw, float) else str(raw).strip()
|
||||
if "e" in text.lower():
|
||||
return 0
|
||||
return len(text.split(".", 1)[1]) if "." in text else 0
|
||||
|
||||
|
||||
def closes(total: Fraction, cell: Fraction, raw: Any) -> bool:
|
||||
"""各段之和(或合起来的比率)和总数那一格对得上:十进制逐位相等;总数取整过的,差在它书写精度的半个末位以内也算;浮点
|
||||
算出来的差在最后几位的也算(相对误差 1e-9 以内,同 engine/compare._equal)。"""
|
||||
if total == cell:
|
||||
return True
|
||||
gap = abs(total - cell)
|
||||
if gap <= Fraction(1, 2) / Fraction(10) ** written_places(raw):
|
||||
return True
|
||||
scale = max(abs(total), abs(cell))
|
||||
return scale != 0 and gap <= scale * Fraction(1, 10 ** 9)
|
||||
|
||||
|
||||
def sign_word(value: Fraction | float | int) -> str | None:
|
||||
return "up" if value > 0 else "down" if value < 0 else None
|
||||
|
||||
|
||||
__all__ = ["DERIVED_PARAMS", "Env", "build", "closes", "column_label", "label_for", "need", "refuse", "sign_word",
|
||||
"today_param", "written_places"]
|
||||
@@ -0,0 +1,540 @@
|
||||
"""可比性检查(细案 01 §6):比两期之前先确认能比。三处用到:agent 单独调用(analysis_comparable);变化拆解给了两份快照时
|
||||
先做;口径卡跨两份查询算环比、同比时自动做(入口经 run 调)。波动基线、趋势判本期完整用这里的 span_coverage。
|
||||
|
||||
五项检查,每项 pass / warn / fail / unknown(unknown 只写「核对不了」,不判不可比):
|
||||
|
||||
- 同口径:两份 SQL 一字不差 → pass;规范化(data/sql_semantics.normalize_sql)后只差字面量:把时间条件的界(data/coverage
|
||||
认出的时间谓词的界,整个表达式)和像日期的字面量换成同一个占位再比,一样就是只差日期 → pass,不一样就是比的两段不是
|
||||
两期 → fail「两边除了日期,筛选的还不一样」;规范形式不同交给口径账本的 explain_sql_diff,等价 → pass,只差时间范围、排序
|
||||
→ pass,否则 fail,写出差在哪些方面(只写方面,不写 SQL 原文里的值)。两份来自不同的数据源 fail。同一份快照里按期次分组
|
||||
的天然同口径。
|
||||
- 窗口长度:用 data/coverage 的 filters 取业务日期列(目录的 business_date;没写就取唯一那一列时间谓词)上的区间,界是字面
|
||||
量的直接解析,按运行当天现算的在库里算(Book 的 bound,记进 probes);算两边天数。整月对整月、整季对整季算一样长。
|
||||
不相等:度量是流量 fail,比率、存量、没标的 warn。按期次分组的:两期粒度相同,一样长。
|
||||
- 整期:窗口(或这一期)的终点和表里这一列的实际最大值(Book 的 range:现查,查不成退回剖析事实)、运行当天比;数据从
|
||||
窗口起点之后才开始的也算不完整。不完整:流量 fail(「本期数据只到 2026-10-15,还差 3 天」),其余 warn;表的范围查不到时
|
||||
只能按运行当天判,判不出的 unknown。
|
||||
- 星期构成:按天的窗口、天数不是 7 的倍数时数两边的周六、周日天数,不同 warn(「两段的周末天数不同:4 天对 6 天」)。整月、
|
||||
整季、按周的不看。
|
||||
- 数据问题:两边查询结果上「查询区间里这张表没有数据」(快照的 no_data)fail,SQL 检查的错误级结果(快照的 checks,level
|
||||
error)warn。
|
||||
|
||||
判定:有 fail → incomparable;只有 warn → conditional;其余 comparable(有核对不了的项照实写在判定语后面)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, timedelta
|
||||
from typing import Any
|
||||
|
||||
from app.analysis import periods as pd
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis.facts import Facts, date_source
|
||||
from app.analysis.inputs import Snap
|
||||
from app.analysis.probe import Book
|
||||
from app.data.names import name_key
|
||||
|
||||
CHECKS = ("same_query", "window", "complete", "weekdays", "data")
|
||||
CHECK_NAMES = {"same_query": "同口径", "window": "窗口长度", "complete": "整期", "weekdays": "星期构成",
|
||||
"data": "数据问题"}
|
||||
STATUS_NAMES = {"pass": "通过", "warn": "提醒", "fail": "不通过", "unknown": "核对不了"}
|
||||
#: explain_sql_diff 的方面 → 叫法
|
||||
_ASPECTS = {"time": "时间列", "time_range": "时间范围", "filter": "筛选条件", "aggregate": "聚合方式",
|
||||
"expression": "选出来的列", "join": "关联", "tables": "查的表", "set_op": "合并方式", "group_by": "分组",
|
||||
"having": "分组后的筛选", "limit": "取哪几行", "order": "排序", "parse": "写法", "other": "写法"}
|
||||
#: 只差这几个方面的两份 SQL 仍是同一个量的两期
|
||||
_SAME_ASPECTS = frozenset({"time_range", "order"})
|
||||
#: 一项 SQL 检查的说明最多引多少字
|
||||
_CHECK_TEXT_MAX = 80
|
||||
|
||||
|
||||
@dataclass
|
||||
class Check:
|
||||
id: str
|
||||
status: str = "unknown"
|
||||
reasons: list[str] = field(default_factory=list)
|
||||
base: str | None = None
|
||||
target: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Span:
|
||||
"""一边的时间范围:start / end 是窗口(或这一期)的头尾(含);table / column 是这段时间按的那一列(探查表的范围用);
|
||||
sql_window 是 SQL 时间条件在这一列上的区间(按期次分组时看它有没有截掉这一期)。"""
|
||||
|
||||
label: str
|
||||
snap: Snap
|
||||
start: date | None = None
|
||||
end: date | None = None
|
||||
table: str | None = None
|
||||
column: str | None = None
|
||||
sql_window: Any = None
|
||||
unit: str | None = None # 这一段是不是整月、整季、整年(calendar_unit),按期次时是粒度
|
||||
|
||||
|
||||
@dataclass
|
||||
class Outcome:
|
||||
checks: list[Check]
|
||||
verdict: str
|
||||
reasons: list[str]
|
||||
unknown: list[str]
|
||||
phrase: str
|
||||
structure: str | None # 同口径不通过的原因(变化拆解据此不拆)
|
||||
base: Span | None = None
|
||||
target: Span | None = None
|
||||
|
||||
def check(self, cid: str) -> Check:
|
||||
return next(c for c in self.checks if c.id == cid)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 同口径
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None:
|
||||
"""时间条件的界、像日期的字面量换成同一个占位之后的规范形式(文本 + 剩下的字面量)。解析不了为 None。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.data import coverage
|
||||
from app.data.sql_semantics import normalize_sql
|
||||
|
||||
tree = coverage.parse_sql(sql, read)
|
||||
if tree is None:
|
||||
return None
|
||||
for flt in coverage.filters(tree):
|
||||
for _, node, _ in flt.conds:
|
||||
if node.parent is not None:
|
||||
node.replace(exp.Literal.string("__day__"))
|
||||
for lit in list(tree.find_all(exp.Literal)):
|
||||
if coverage.literal_bound(lit)[1] is not None:
|
||||
lit.replace(exp.Literal.string("__day__"))
|
||||
normalized = normalize_sql(tree.sql(dialect=read), read)
|
||||
if normalized.parse_error:
|
||||
return None
|
||||
return normalized.text, normalized.literals
|
||||
|
||||
|
||||
def same_query(base: Snap, target: Snap) -> Check:
|
||||
from app.data.sql_semantics import explain_sql_diff, normalize_sql
|
||||
|
||||
check = Check("same_query")
|
||||
if base.source != target.source:
|
||||
check.status, check.reasons = "fail", ["两份查询来自不同的数据源"]
|
||||
return check
|
||||
a, b = base.sql.strip(), target.sql.strip()
|
||||
if a == b:
|
||||
check.status = "pass"
|
||||
return check
|
||||
read = base.dialect or target.dialect
|
||||
na, nb = normalize_sql(a, read), normalize_sql(b, read)
|
||||
if na.parse_error or nb.parse_error:
|
||||
check.reasons = ["SQL 解析不了"]
|
||||
return check
|
||||
if na.text == nb.text:
|
||||
ua, ub = _undated(a, read), _undated(b, read)
|
||||
if ua is None or ub is None:
|
||||
return check
|
||||
if ua == ub:
|
||||
check.status = "pass"
|
||||
else:
|
||||
check.status, check.reasons = "fail", ["两边除了日期,筛选的还不一样"]
|
||||
return check
|
||||
try:
|
||||
diff = explain_sql_diff(a, b, read)
|
||||
except Exception: # noqa: BLE001 - 比不了就当核对不了
|
||||
return check
|
||||
aspects = [d.aspect for d in diff.items]
|
||||
if diff.equivalent or (aspects and set(aspects) <= _SAME_ASPECTS):
|
||||
check.status = "pass"
|
||||
return check
|
||||
names = list(dict.fromkeys(_ASPECTS.get(x, "写法") for x in aspects if x not in _SAME_ASPECTS)) or ["写法"]
|
||||
check.status, check.reasons = "fail", [f"两份查询的口径结构不同(差在{'、'.join(names)})"]
|
||||
return check
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 窗口
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _bound_days(snap: Snap, conds: Sequence[tuple[str, Any, bool]], book: Book) -> dict[int, Any]:
|
||||
from app.data import coverage
|
||||
|
||||
days: dict[int, Any] = {}
|
||||
for _, node, _ in conds:
|
||||
literal, day = coverage.literal_bound(node)
|
||||
if literal:
|
||||
days[id(node)] = day
|
||||
continue
|
||||
try:
|
||||
text = node.sql(dialect=snap.dialect)
|
||||
except Exception: # noqa: BLE001
|
||||
days[id(node)] = None
|
||||
continue
|
||||
probe = book.bound(snap.source, text)
|
||||
days[id(node)] = coverage.as_day(probe.get("value")) if probe else None
|
||||
return days
|
||||
|
||||
|
||||
def _filters_on(snap: Snap, measure: str | None, facts: Facts,
|
||||
column: tuple[str, str] | None = None) -> tuple[Any, str | None]:
|
||||
"""SQL(收窄到度量那一项)里按业务日期列(或唯一那一列时间谓词,或给定的那一列)的时间条件:(那段条件, 认不出的原因)。"""
|
||||
from app.data import coverage
|
||||
|
||||
tree = coverage.parse_sql(snap.sql, snap.dialect)
|
||||
if tree is None:
|
||||
return None, "SQL 解析不了"
|
||||
narrowed = coverage.narrowed(tree, measure, snap.dialect) if measure else tree
|
||||
flts = coverage.filters(narrowed)
|
||||
groups: dict[tuple[str, str], list[Any]] = {}
|
||||
for flt in flts:
|
||||
groups.setdefault((name_key(flt.table_name), name_key(flt.column_name)), []).append(flt)
|
||||
if column is not None:
|
||||
wanted = (name_key(column[0]), name_key(column[1]))
|
||||
wanted_short = (name_key(column[0].rsplit(".", 1)[-1]), wanted[1])
|
||||
chosen = groups.get(wanted) or next((v for k, v in groups.items()
|
||||
if (k[0].rsplit(".", 1)[-1], k[1]) == wanted_short), None)
|
||||
if chosen is None:
|
||||
return None, None
|
||||
else:
|
||||
if not groups:
|
||||
return None, "SQL 里没有时间条件"
|
||||
dated = [v for v in groups.values()
|
||||
if (bd := facts.business_date(snap.source, v[0].table_name)) and name_key(bd) == name_key(v[0].column_name)]
|
||||
if len(dated) == 1:
|
||||
chosen = dated[0]
|
||||
elif len(groups) == 1:
|
||||
chosen = next(iter(groups.values()))
|
||||
else:
|
||||
return None, "时间条件不止一列,认不出按哪一列算"
|
||||
if len({f.key for f in chosen}) > 1:
|
||||
return None, "这一列上有好几段时间条件"
|
||||
return chosen[0], None
|
||||
|
||||
|
||||
def sql_span(label: str, snap: Snap, measure: str | None, facts: Facts, book: Book) -> tuple[Span, str | None]:
|
||||
"""一份查询的时间窗口(两份快照比两期时)。"""
|
||||
from app.data import coverage
|
||||
|
||||
span = Span(label, snap)
|
||||
flt, why = _filters_on(snap, measure, facts)
|
||||
if flt is None:
|
||||
return span, why
|
||||
if flt.column_name.lower() in snap.masked:
|
||||
return span, "时间列是遮罩列" # 遮罩列不探表的范围
|
||||
span.table, span.column = flt.table_name, flt.column_name
|
||||
window = coverage.window_of(flt.conds, _bound_days(snap, flt.conds, book))
|
||||
if window is None or window.start is None or window.end is None:
|
||||
return span, "时间窗口没有两头"
|
||||
span.start, span.end = window.start, window.end
|
||||
span.unit = pd.calendar_unit(window.start, window.end)
|
||||
return span, None
|
||||
|
||||
|
||||
def period_span(label: str, snap: Snap, period: Any, item: pd.Period, grain: str, measure: str | None,
|
||||
facts: Facts, book: Book) -> Span:
|
||||
"""按期次分组的一份快照里的一期:这一期覆盖的日子,加上 SQL 在期次那一列上的时间条件。"""
|
||||
from app.data import coverage
|
||||
|
||||
span = Span(label, snap, item.start, item.end, unit=grain)
|
||||
found = date_source(snap.sql, period)
|
||||
if found is None:
|
||||
return span
|
||||
span.table, span.column = found
|
||||
if span.column.lower() in snap.masked:
|
||||
span.table = span.column = None
|
||||
return span
|
||||
flt, _ = _filters_on(snap, measure, facts, column=found)
|
||||
if flt is not None:
|
||||
span.sql_window = coverage.window_of(flt.conds, _bound_days(snap, flt.conds, book))
|
||||
return span
|
||||
|
||||
|
||||
def _range(span: Span, book: Book) -> tuple[bool, date | None, date | None, bool]:
|
||||
"""(查得到, 最早, 最晚, 一个值都没有):表里这一列的实际范围。"""
|
||||
from app.data import coverage
|
||||
|
||||
if not (span.table and span.column):
|
||||
return False, None, None, False
|
||||
probe = book.range(span.snap.source, span.table, span.column)
|
||||
if probe is None:
|
||||
return False, None, None, False
|
||||
lo = coverage.as_day(probe.get("min")) if probe.get("min") else None
|
||||
hi = coverage.as_day(probe.get("max")) if probe.get("max") else None
|
||||
empty = probe.get("basis") == "live" and probe.get("min") is None and probe.get("max") is None
|
||||
return True, lo.day if lo else None, hi.day if hi else None, empty
|
||||
|
||||
|
||||
def span_coverage(span: Span, today: date, book: Book) -> tuple[str, str | None]:
|
||||
"""这一段(窗口或一期)的数据齐不齐:(pass / short / unknown, 不齐的那一句)。没过完(终点不早于运行当天:当天的数据还没
|
||||
到齐)、SQL 的时间条件截掉了一部分、表里的数据没到(最晚早于终点)或来得晚(最早晚于起点),都算不齐。"""
|
||||
if span.start is None or span.end is None:
|
||||
return "unknown", None
|
||||
lo, hi = span.start, span.end
|
||||
window = span.sql_window
|
||||
if window is not None:
|
||||
if window.start is not None:
|
||||
lo = max(lo, window.start)
|
||||
if window.end is not None:
|
||||
hi = min(hi, window.end)
|
||||
if span.end >= today:
|
||||
hi = min(hi, today - timedelta(days=1))
|
||||
known, dmin, dmax, empty = _range(span, book)
|
||||
if empty:
|
||||
return "short", f"{span.label}在 {span.start.isoformat()} 到 {span.end.isoformat()} 没有数据"
|
||||
if dmin is not None:
|
||||
lo = max(lo, dmin)
|
||||
if dmax is not None:
|
||||
hi = min(hi, dmax)
|
||||
if hi < lo:
|
||||
return "short", f"{span.label}在 {span.start.isoformat()} 到 {span.end.isoformat()} 没有数据"
|
||||
parts = []
|
||||
if hi < span.end:
|
||||
parts.append(f"{span.label}数据只到 {hi.isoformat()},还差 {(span.end - hi).days} 天")
|
||||
if lo > span.start:
|
||||
parts.append(f"{span.label}数据从 {lo.isoformat()} 才开始,缺前 {(lo - span.start).days} 天")
|
||||
if parts:
|
||||
return "short", ",".join(parts)
|
||||
return ("pass", None) if known and dmax is not None else ("unknown", None)
|
||||
|
||||
|
||||
def _days(span: Span) -> int | None:
|
||||
return (span.end - span.start).days + 1 if span.start and span.end else None
|
||||
|
||||
|
||||
def _shown(span: Span) -> str | None:
|
||||
if span.start is None or span.end is None:
|
||||
return None
|
||||
if span.start == span.end:
|
||||
return f"{span.start.isoformat()}(1 天)"
|
||||
return f"{span.start.isoformat()} 到 {span.end.isoformat()}({_days(span)} 天)"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 五项检查和判定
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _window_check(base: Span, target: Span, flow: bool, *, grouped: bool) -> Check:
|
||||
check = Check("window", base=_shown(base), target=_shown(target))
|
||||
a, b = _days(base), _days(target)
|
||||
if a is None or b is None:
|
||||
return check
|
||||
if a == b or grouped or (base.unit and base.unit == target.unit):
|
||||
check.status = "pass"
|
||||
return check
|
||||
check.status = "fail" if flow else "warn"
|
||||
check.reasons = [f"两段的天数不同:{a} 天对 {b} 天"]
|
||||
return check
|
||||
|
||||
|
||||
def _complete_check(sides: Sequence[Span], today: date, book: Book, flow: bool) -> Check:
|
||||
check = Check("complete")
|
||||
shorts, unknown = [], False
|
||||
for span in sides:
|
||||
status, reason = span_coverage(span, today, book)
|
||||
if status == "short":
|
||||
shorts.append(reason)
|
||||
elif status == "unknown":
|
||||
unknown = True
|
||||
if span.label == "基期":
|
||||
check.base = reason or STATUS_NAMES[status if status != "short" else "fail"]
|
||||
else:
|
||||
check.target = reason or STATUS_NAMES[status if status != "short" else "fail"]
|
||||
if shorts:
|
||||
check.status, check.reasons = ("fail" if flow else "warn"), [r for r in shorts if r]
|
||||
elif not unknown:
|
||||
check.status = "pass"
|
||||
return check
|
||||
|
||||
|
||||
def _weekday_check(base: Span, target: Span, *, grain: str | None) -> Check:
|
||||
check = Check("weekdays")
|
||||
a, b = _days(base), _days(target)
|
||||
if a is None or b is None:
|
||||
return check
|
||||
if grain in ("week", "month", "quarter", "seq") or (a % 7 == 0 and b % 7 == 0) or \
|
||||
(base.unit and base.unit == target.unit and base.unit in ("month", "quarter", "year")):
|
||||
check.status = "pass"
|
||||
return check
|
||||
wa, wb = pd.weekend_days(base.start, base.end), pd.weekend_days(target.start, target.end) # type: ignore[arg-type]
|
||||
check.base, check.target = f"周末 {wa} 天", f"周末 {wb} 天"
|
||||
if wa == wb:
|
||||
check.status = "pass"
|
||||
else:
|
||||
check.status, check.reasons = "warn", [f"两段的周末天数不同:{wa} 天对 {wb} 天"]
|
||||
return check
|
||||
|
||||
|
||||
def _data_check(sides: Sequence[tuple[str, Snap]], measure: str | None) -> Check:
|
||||
from app.data import coverage
|
||||
|
||||
check = Check("data", status="pass")
|
||||
fails, warns = [], []
|
||||
for label, snap in sides:
|
||||
found = coverage.no_data_of(snap.data, measure) if measure else \
|
||||
next(iter(snap.data.get("no_data") or []), None)
|
||||
if found:
|
||||
fails.append(f"{label}查询的区间里这张表没有数据")
|
||||
errors = [c for c in snap.data.get("checks") or [] if isinstance(c, Mapping) and c.get("level") == "error"]
|
||||
if errors:
|
||||
text = str(errors[0].get("message") or "").strip().replace(";", ",")
|
||||
if len(text) > _CHECK_TEXT_MAX:
|
||||
text = text[:_CHECK_TEXT_MAX - 1] + "…"
|
||||
warns.append(f"{label}查询的 SQL 检查有错误:{text}" if text else f"{label}查询的 SQL 检查有错误")
|
||||
if fails:
|
||||
check.status = "fail"
|
||||
elif warns:
|
||||
check.status = "warn"
|
||||
check.reasons = fails + warns
|
||||
return check
|
||||
|
||||
|
||||
def _decide(checks: list[Check], base: Span | None, target: Span | None) -> Outcome:
|
||||
fails = [r for c in checks if c.status == "fail" for r in c.reasons]
|
||||
warns = [r for c in checks if c.status == "warn" for r in c.reasons]
|
||||
unknown = [CHECK_NAMES[c.id] for c in checks if c.status == "unknown"]
|
||||
verdict = "incomparable" if fails else "conditional" if warns else "comparable"
|
||||
reasons = fails + warns
|
||||
structure = next((c.reasons[0] for c in checks if c.id == "same_query" and c.status == "fail" and c.reasons), None)
|
||||
return Outcome(checks, verdict, reasons, unknown, ph.comparable_phrase(verdict, reasons, unknown), structure,
|
||||
base, target)
|
||||
|
||||
|
||||
def compare_queries(base: Snap, target: Snap, measure: str | None, facts: Facts, book: Book, today: date) -> Outcome:
|
||||
"""两份快照比两期(本周一份、上周一份)。measure 是两边的度量列(收窄 SQL 找时间条件、认度量类型);没有为 None。"""
|
||||
same = same_query(base, target)
|
||||
b_span, _ = sql_span("基期", base, measure, facts, book)
|
||||
t_span, _ = sql_span("本期", target, measure, facts, book)
|
||||
flow = bool(measure) and facts.measure(target, measure)[0] == "flow" # type: ignore[arg-type]
|
||||
checks = [same, _window_check(b_span, t_span, flow, grouped=False),
|
||||
_complete_check([b_span, t_span], today, book, flow), _weekday_check(b_span, t_span, grain=None),
|
||||
_data_check([("基期", base), ("本期", target)], measure)]
|
||||
return _decide(checks, b_span, t_span)
|
||||
|
||||
|
||||
def compare_periods(snap: Snap, period: str, base: pd.Period, target: pd.Period, grain: str, measure: str | None,
|
||||
facts: Facts, book: Book, today: date) -> Outcome:
|
||||
"""同一份快照里按期次分组的两期:天然同口径;窗口按期次的粒度;整期看这两期各自齐不齐。"""
|
||||
same = Check("same_query", status="pass")
|
||||
b_span = period_span("基期", snap, period, base, grain, measure, facts, book)
|
||||
t_span = period_span("本期", snap, period, target, grain, measure, facts, book)
|
||||
flow = bool(measure) and facts.measure(snap, measure)[0] == "flow" # type: ignore[arg-type]
|
||||
if grain == "seq":
|
||||
checks = [same, Check("window"), Check("complete"), Check("weekdays"), _data_check([("", snap)], measure)]
|
||||
else:
|
||||
checks = [same, _window_check(b_span, t_span, flow, grouped=True),
|
||||
_complete_check([b_span, t_span], today, book, flow),
|
||||
_weekday_check(b_span, t_span, grain=grain), _data_check([("", snap)], measure)]
|
||||
return _decide(checks, b_span, t_span)
|
||||
|
||||
|
||||
def table_of(outcome: Outcome) -> dict[str, Any]:
|
||||
"""检查表(结论的 table):检查项、结果、基期、本期、说明。"""
|
||||
rows = [[CHECK_NAMES[c.id], STATUS_NAMES[c.status], c.base, c.target, ";".join(c.reasons) or None]
|
||||
for c in outcome.checks]
|
||||
return {"columns": ["check", "status", "base", "target", "note"],
|
||||
"labels": {"check": "检查项", "status": "结果", "base": "基期", "target": "本期", "note": "说明"},
|
||||
"rows": rows}
|
||||
|
||||
|
||||
def values_of(outcome: Outcome) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {c.id: c.status for c in outcome.checks}
|
||||
for side, span in (("base", outcome.base), ("target", outcome.target)):
|
||||
if span is not None and span.start and span.end:
|
||||
out[f"{side}_start"] = span.start.isoformat()
|
||||
out[f"{side}_end"] = span.end.isoformat()
|
||||
out[f"{side}_days"] = str(_days(span))
|
||||
out[f"{side}_weekend"] = str(pd.weekend_days(span.start, span.end))
|
||||
return out
|
||||
|
||||
|
||||
def reasons(result: Mapping[str, Any]) -> list[str]:
|
||||
"""一份可比性结论里 fail、warn 的原因,一句一个(口径卡指标的 comparability.reasons 用)。按检查表的说明拆开。"""
|
||||
out: list[str] = []
|
||||
table = result.get("table") or {}
|
||||
for row in table.get("rows") or []:
|
||||
if len(row) >= 5 and row[1] in (STATUS_NAMES["fail"], STATUS_NAMES["warn"]) and row[4]:
|
||||
out.extend(str(row[4]).split(";"))
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 算子:analysis_comparable
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def comparable(raw: Mapping[str, Any], env: Any) -> Any:
|
||||
"""可比性检查这个算子。参数:两份快照 base_query、target_query;或一份快照 query + period(期次列)+ base、target(期次
|
||||
列里的两个值)、可选 grain。都可以写 measure(度量列:收窄 SQL 找时间条件、按目录认它是不是流量;不写时只有一列数值列
|
||||
就用它)。"""
|
||||
from app.analysis.common import build, need, today_param
|
||||
from app.analysis.contract import CallError
|
||||
from app.analysis.inputs import same_value
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
two = raw.get("base_query") is not None or raw.get("target_query") is not None
|
||||
if two:
|
||||
base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号")
|
||||
target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号")
|
||||
if base is target:
|
||||
from app.analysis.decompose import SAME_QUERY
|
||||
|
||||
raise CallError("same_query", SAME_QUERY)
|
||||
params.update(base_query=base.ref, target_query=target.ref)
|
||||
else:
|
||||
base = target = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
|
||||
params["query"] = base.ref
|
||||
measure = target.column(raw["measure"], "度量列") if raw.get("measure") is not None else _only_number(target)
|
||||
if measure is not None and two:
|
||||
base.column(measure, "度量列")
|
||||
params["measure"] = measure
|
||||
params["today"] = today_param(env)
|
||||
if two:
|
||||
env.inputs.use(base, "base", [measure] if measure else [])
|
||||
env.inputs.use(target, "target", [measure] if measure else [])
|
||||
outcome = compare_queries(base, target, measure, env.facts, env.book, env.today)
|
||||
else:
|
||||
period = target.column(need(raw, "period", "期次列"), "期次列")
|
||||
grain = raw.get("grain")
|
||||
if grain is not None and grain not in (*pd.GRAINS, "seq"):
|
||||
raise CallError("bad_grain", "粒度只能写 day、week、month、quarter,或者不写(按期次值推断)")
|
||||
params.update(period=period, base=need(raw, "base", "基期(期次列里的值)"),
|
||||
target=need(raw, "target", "本期(期次列里的值)"))
|
||||
i = target.index(period)
|
||||
values = [row[i] if i < len(row) else None for row in target.rows]
|
||||
rows = {side: [r for r, v in enumerate(values) if same_value(params[side], v)] for side in ("base", "target")}
|
||||
for side, what in (("base", "基期"), ("target", "本期")):
|
||||
if not rows[side]:
|
||||
raise CallError("period_not_found", f"期次列「{period}」里没有「{params[side]}」:{what}要写 {target.ref} "
|
||||
"里有的期次值")
|
||||
env.inputs.use(target, "data", [period, *([measure] if measure else [])], rows["base"] + rows["target"])
|
||||
parsed = pd.parse(values, grain)
|
||||
if parsed is None:
|
||||
item_b = item_t = pd.Period(None, 0, None, None)
|
||||
grain = "seq"
|
||||
else:
|
||||
items, grain = parsed
|
||||
item_b, item_t = items[rows["base"][0]], items[rows["target"][0]]
|
||||
params["grain"] = grain
|
||||
outcome = compare_periods(target, period, item_b, item_t, grain, measure, env.facts, env.book, env.today)
|
||||
caveats = []
|
||||
window = outcome.check("window")
|
||||
if measure and window.status == "warn" and env.facts.measure(target, measure)[0] is None:
|
||||
caveats.append(ph.caveat("measure_unlabeled", ph.unlabeled_text(measure, "窗口不等只作提醒")))
|
||||
verdict = {"code": outcome.verdict, "subjects": [], "direction": None}
|
||||
return build(env, params, [], table=table_of(outcome), values=values_of(outcome), verdict=verdict,
|
||||
criterion=ph.CRITERION_COMPARABLE, phrase=outcome.phrase, caveats=caveats)
|
||||
|
||||
|
||||
def _only_number(snap: Snap) -> str | None:
|
||||
"""没写度量列时:快照里唯一一列(没遮罩的)数值列;没有或不止一列为 None。"""
|
||||
found = [c for c in snap.columns if c.lower() not in snap.masked and snap.kind(c) == "number"]
|
||||
return found[0] if len(found) == 1 else None
|
||||
|
||||
|
||||
__all__ = ["CHECKS", "CHECK_NAMES", "Check", "Outcome", "STATUS_NAMES", "Span", "comparable", "compare_periods",
|
||||
"compare_queries", "period_span", "reasons", "same_query", "span_coverage", "sql_span", "table_of",
|
||||
"values_of"]
|
||||
@@ -188,6 +188,12 @@ class InputSnapshot(TypedDict, total=False):
|
||||
rows: list[list[Any]]
|
||||
truncated: bool
|
||||
masked: list[str] # 不能当维度、度量的列(遮罩列、按 SQL 认出改了名字的遮罩列),比较时不分大小写
|
||||
# 以下几项照查询快照里的原样带上(第 1 波 W1-a 加,都可选):
|
||||
dialect: str # SQL 的 sqlglot 方言(sqlite / mysql / postgres / oracle);不给时按探查器的 dialect(数据源名)
|
||||
checks: list[dict[str, Any]] # 快照里的 SQL 检查结果(level 为 error 的算可比性的数据问题)
|
||||
no_data: list[dict[str, Any]] # 快照里的「查询区间里这张表没有数据」
|
||||
mask_columns: list[str] # 没给 masked 时(复算取回的是快照工件本身)按这两项认遮罩列
|
||||
masked_at_query: list[str]
|
||||
|
||||
|
||||
class AnalysisInput(TypedDict, total=False):
|
||||
@@ -201,6 +207,8 @@ class AnalysisInput(TypedDict, total=False):
|
||||
row: int # 一格时:第几行、哪一列、值(十进制字符串)
|
||||
column: str
|
||||
value: str | None
|
||||
dialect: str # 读这份快照的 SQL 用的方言(第 1 波 W1-a 加,可选):复算按它读,界的写法才一样
|
||||
masked: list[str] # 这份快照当时不能当维度、度量的列(小写,只记列名;第 1 波 W1-a 加,可选):复算按它认
|
||||
|
||||
|
||||
class Probe(TypedDict, total=False):
|
||||
@@ -213,6 +221,10 @@ class Probe(TypedDict, total=False):
|
||||
max: str | None
|
||||
basis: Literal["live", "profile", "observed"]
|
||||
at: str # 现查的时刻(ISO,服务端时区)
|
||||
# 按运行当天现算的界(CURRENT_DATE、date('now', …))在库里算出的值(第 1 波 W1-a 加,可选):有 expr 的这一条记的是界,
|
||||
# 没有 table / column / min / max
|
||||
expr: str # 界的 SQL 写法(按这份快照的方言)
|
||||
value: str | None # 库里算出的日期(带时刻的写成 YYYY-MM-DD HH:MM:SS)
|
||||
|
||||
|
||||
class Verdict(TypedDict, total=False):
|
||||
@@ -284,7 +296,7 @@ class Context(TypedDict, total=False):
|
||||
today: date # 运行当天(服务端时区):判本期完不完整
|
||||
probe: Prober | None # None 时只用 facts
|
||||
catalogs: Mapping[str, Any] # 数据源名 → 数据目录(app/data/catalog 的 notes):认度量类型、业务日期列、列的叫法
|
||||
facts: Mapping[str, Any] # 数据源名 → 剖析事实(探查失败时兜底)
|
||||
facts: Mapping[str, Any] # 数据源名 → 剖析事实(探查失败时兜底):{表名: last_facts 的那一项},或取它的函数
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,732 @@
|
||||
"""变化拆解(细案 01 §2):变化从哪来。三种模式,agent 按指标的性质选,算子核对选得对不对。
|
||||
|
||||
**参数**(AnalysisRequest.params;列名不分大小写,按快照认):
|
||||
|
||||
- 一份快照:query(编号)、period(期次列)、base / target(期次列里的两个值);或两份快照:base_query、target_query(各自
|
||||
只有维度列和度量列,先做可比性检查:口径结构不同的不拆,算不了 incomparable_sides;其余不可比的照样算,判定后附上
|
||||
可比性结论)。
|
||||
- additive:by(维度列,1–2 列)、measure(度量列);可选 headline {base, target}(基期、本期总数各一格,如 Q1.r1.gmv,用来
|
||||
核对闭合)、parent {of: A3, key: [华东]}(下钻:这一层各段之和要等于 A3 里华东那一段)。
|
||||
- ratio:by、numerator(分子列)、denominator(分母列);可选 headline {base, target}(整体比率的两格)、parent(上一层的比率
|
||||
拆解)。
|
||||
- factor:product(乘积列)、factors(因子列,2–4 个);一份快照按 period / base / target 取两行,两份快照各一行。
|
||||
- 都可以写 label(度量的叫法,消毒后进判定语)。
|
||||
|
||||
**计算**全程 Fraction(快照里的数按字面换成精确的有理数):各段之和逐位等于总变化;比率拆解的结构效应、自身效应按中点
|
||||
对称拆,Σmix + Σrate 逐位等于比率的变化;乘积拆解是 LMDI-I(对数),浮点,各因子贡献之和在浮点误差内等于总变化。判定、
|
||||
阈值、边界照细案 §2.1–§2.4。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
from app.analysis import comparable as cmp
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis.common import Env, build, closes, column_label, label_for, need, refuse, sign_word, today_param
|
||||
from app.analysis.contract import CallError
|
||||
from app.analysis.inputs import Snap, as_list, hashable, same_value
|
||||
from app.analysis.numbers import canon, to_fraction
|
||||
|
||||
#: 辛普森:分母够大的段里,按平均权重算有这么多的段比率和整体反着变
|
||||
SIMPSON_SHARE = Fraction(4, 5)
|
||||
#: 下钻、总数的写法(调用错误里教 agent 怎么写)
|
||||
PARENT_SHAPE = "下钻要写成 {of: A3, key: [华东]} 这样:of 是上一层变化拆解的编号,key 是那一段的维度值"
|
||||
HEADLINE_SHAPE = "总数要写成 {base: Q1.r1.gmv, target: Q1.r0.gmv} 这样:base 是基期总数那一格,target 是本期总数那一格"
|
||||
SAME_QUERY = "基期和本期是同一份查询:两期各给一份查询,或者给一份按期次分组的查询和期次列"
|
||||
#: 乘积闭合的最小相对误差
|
||||
_FACTOR_REL = Fraction(1, 10 ** 6)
|
||||
_NOT_ADDITIVE = {"ratio": "比率", "identifier": "标识", "status": "状态", "attribute": "属性"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Side:
|
||||
"""一期的各段:键 → 值(ratio 模式是 (分子, 分母));nulls 是这一期度量格为空的段;rows 是用到的行。"""
|
||||
|
||||
snap: Snap
|
||||
values: dict[tuple[Any, ...], Any] = field(default_factory=dict)
|
||||
nulls: list[tuple[Any, ...]] = field(default_factory=list)
|
||||
rows: list[int] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Segment:
|
||||
key: tuple[Any, ...]
|
||||
base: Any
|
||||
target: Any
|
||||
flags: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def _key_strings(key: Sequence[Any]) -> list[str]:
|
||||
return [ph.key_text([k]) for k in key]
|
||||
|
||||
|
||||
def _sides(raw: Mapping[str, Any], env: Env, roles: Sequence[tuple[str, str]], *, by_needed: bool = True
|
||||
) -> tuple[dict[str, Any], Snap, Snap, dict[str, str], list[str]]:
|
||||
"""一份或两份快照:(规范化的参数, 基期快照, 本期快照, {角色: 列名}, 维度列)。roles 是 [(参数名, 角色的叫法)]。"""
|
||||
params: dict[str, Any] = {}
|
||||
two = raw.get("base_query") is not None or raw.get("target_query") is not None
|
||||
if two:
|
||||
base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号")
|
||||
target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号")
|
||||
if base is target:
|
||||
raise CallError("same_query", SAME_QUERY)
|
||||
params.update(base_query=base.ref, target_query=target.ref)
|
||||
else:
|
||||
base = target = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
|
||||
params["query"] = base.ref
|
||||
by: list[str] = []
|
||||
if by_needed:
|
||||
wanted = as_list(raw.get("by"))
|
||||
if not 1 <= len(wanted) <= 2:
|
||||
raise CallError("bad_by", "维度列要写 1 到 2 列")
|
||||
by = [target.column(c, "维度列") for c in wanted]
|
||||
if two:
|
||||
for c in by:
|
||||
base.column(c, "维度列")
|
||||
params["by"] = by
|
||||
cols: dict[str, str] = {}
|
||||
for key, what in roles:
|
||||
if key == "factors":
|
||||
continue
|
||||
col = target.column(need(raw, key, what), what)
|
||||
if two:
|
||||
base.column(col, what)
|
||||
cols[key] = col
|
||||
params[key] = col
|
||||
if not two:
|
||||
params["period"] = target.column(need(raw, "period", "期次列"), "期次列")
|
||||
params["base"] = need(raw, "base", "基期(期次列里的值)")
|
||||
params["target"] = need(raw, "target", "本期(期次列里的值)")
|
||||
if same_value(params["base"], params["target"]):
|
||||
raise CallError("same_period", "基期和本期是同一期:写期次列里两个不同的值")
|
||||
used = [*cols.values(), *([params["period"]] if "period" in params else [])]
|
||||
if len(set(by)) != len(by) or set(by) & set(used) or len(set(used)) != len(used):
|
||||
raise CallError("bad_by", "维度列、期次列、度量列要是不同的列")
|
||||
return params, base, target, cols, by
|
||||
|
||||
|
||||
def _period_rows(snap: Snap, period: str, wanted: Any, what: str) -> list[int]:
|
||||
i = snap.index(period)
|
||||
rows = [r for r, row in enumerate(snap.rows) if i < len(row) and same_value(wanted, row[i])]
|
||||
if not rows:
|
||||
values = list(dict.fromkeys(ph.key_text([row[i]]) for row in snap.rows if i < len(row)))
|
||||
raise CallError("period_not_found", f"期次列「{period}」里没有「{wanted}」:{what}要写 {snap.ref} 里有的期次值,"
|
||||
f"比如 {'、'.join(values[:6])}")
|
||||
return rows
|
||||
|
||||
|
||||
def _rows_of(raw: Mapping[str, Any], params: dict[str, Any], base: Snap, target: Snap) -> tuple[list[int], list[int]]:
|
||||
if "period" in params:
|
||||
return (_period_rows(base, params["period"], params["base"], "基期"),
|
||||
_period_rows(target, params["period"], params["target"], "本期"))
|
||||
return list(range(len(base.rows))), list(range(len(target.rows)))
|
||||
|
||||
|
||||
def _collect(snap: Snap, rows: Sequence[int], by: Sequence[str], read: Any, labels: Sequence[str],
|
||||
period_label: str | None) -> Side:
|
||||
"""一期的各段:按维度键,同一键只许一行(多行是 SQL 没按这些维度汇总)。read(row) 读出这一行的值(None 是空)。"""
|
||||
side = Side(snap)
|
||||
seen: set[tuple[Any, ...]] = set()
|
||||
idx = [snap.index(c) for c in by]
|
||||
for r in rows:
|
||||
row = snap.rows[r]
|
||||
key = tuple(hashable(row[i] if i < len(row) else None) for i in idx)
|
||||
if key in seen:
|
||||
keys = "、".join([*labels, *([period_label] if period_label else [])])
|
||||
raise CallError("duplicate_rows", f"{snap.ref} 里同一个{'、'.join(labels)}在同一期有不止一行:在 SQL 里按 "
|
||||
f"{keys} 汇总,每段每期一行")
|
||||
seen.add(key)
|
||||
value = read(r)
|
||||
side.rows.append(r)
|
||||
if value is None:
|
||||
side.nulls.append(key)
|
||||
side.values[key] = value
|
||||
return side
|
||||
|
||||
|
||||
def _segments(base: Side, target: Side, zero: Any) -> list[Segment]:
|
||||
"""两期的各段,按先出现的顺序:缺的一边记 0,标「新出现 / 消失」;度量格为空的记 0,标「为空」。"""
|
||||
out: list[Segment] = []
|
||||
for key in dict.fromkeys([*base.values, *target.values]):
|
||||
flags = []
|
||||
if key not in base.values:
|
||||
flags.append("新出现")
|
||||
if key not in target.values:
|
||||
flags.append("消失")
|
||||
if key in base.nulls or key in target.nulls:
|
||||
flags.append("为空")
|
||||
b, t = base.values.get(key), target.values.get(key)
|
||||
out.append(Segment(key, zero if b is None else b, zero if t is None else t, flags))
|
||||
return out
|
||||
|
||||
|
||||
def _parent(raw: Mapping[str, Any], env: Env, mode: str, dims: int) -> tuple[dict[str, Any], list[Any], int]:
|
||||
"""下钻:上一层的结论和那一段。返回 (params 里的 parent, 那一段在上一层结果表里的那一行, 这一层的层数)。"""
|
||||
from app.core.artifact_store import canonical_json, content_hash
|
||||
|
||||
spec = raw.get("parent")
|
||||
name = spec.get("of") if isinstance(spec, Mapping) else None
|
||||
key = as_list(spec.get("key")) if isinstance(spec, Mapping) else []
|
||||
if not isinstance(name, str) or not name.strip() or not key:
|
||||
raise CallError("bad_parent", PARENT_SHAPE)
|
||||
name = name.strip()
|
||||
found = env.parents(name) if env.parents is not None else None
|
||||
if not isinstance(found, Mapping):
|
||||
raise CallError("parent_missing", f"没有叫「{name}」的分析结论:下钻写这次运行里变化拆解的编号(如 A3)")
|
||||
if found.get("op") != "decompose" or found.get("mode") != mode:
|
||||
raise CallError("parent_not_decompose", f"{name} 不是{'按维度' if mode == 'additive' else '比率'}的变化拆解,"
|
||||
"下钻要接在同一种拆解后面")
|
||||
if found.get("status") != "ok" or not isinstance(found.get("table"), Mapping):
|
||||
raise CallError("parent_refused", f"{name} 没有做成,不能在它下面下钻")
|
||||
depth = int((found.get("params") or {}).get("depth") or 0) + 1
|
||||
limit = int(env.settings["max_drill_depth"])
|
||||
if depth > limit:
|
||||
raise CallError("too_deep", f"已经下钻了 {depth - 1} 层,最多 {limit} 层:换一个维度从头拆")
|
||||
table = found["table"]
|
||||
fixed = _FIXED_ADDITIVE if mode == "additive" else _FIXED_RATIO
|
||||
columns = list(table.get("columns") or [])
|
||||
dim_count = len([c for c in columns if c not in fixed])
|
||||
if len(key) != dim_count:
|
||||
raise CallError("bad_parent", f"{name} 按 {dim_count} 列拆,下钻的那一段要写 {dim_count} 个维度值")
|
||||
hit = next((row for row in table.get("rows") or []
|
||||
if all(same_value(k, v) for k, v in zip(key, row[:dim_count]))), None)
|
||||
if hit is None:
|
||||
raise CallError("parent_key_missing", f"{name} 里没有「{ph.key_text(key)}」这一段:写 {name} 结果表里的段")
|
||||
env.inputs.parent(name, content_hash(canonical_json(found)))
|
||||
return {"of": name, "key": key}, dict(zip(columns, hit)), depth
|
||||
|
||||
|
||||
_FIXED_ADDITIVE = ("base", "target", "delta", "pct", "share", "flag")
|
||||
_FIXED_RATIO = ("n_base", "n_target", "d_base", "d_target", "w_base", "w_target", "r_base", "r_target", "mix", "rate",
|
||||
"total", "flag")
|
||||
|
||||
|
||||
def _headline(raw: Mapping[str, Any], env: Env) -> tuple[dict[str, str] | None, dict[str, tuple[Fraction, Any]]]:
|
||||
"""headline:基期、本期总数各一格 → ({base, target} 的写法, {base / target: (值, 原样)})。"""
|
||||
spec = raw.get("headline")
|
||||
if spec is None:
|
||||
return None, {}
|
||||
if isinstance(spec, (list, tuple)) and len(spec) == 2:
|
||||
spec = {"base": spec[0], "target": spec[1]}
|
||||
if not isinstance(spec, Mapping) or not spec.get("base") or not spec.get("target"):
|
||||
raise CallError("bad_headline", HEADLINE_SHAPE)
|
||||
out: dict[str, tuple[Fraction, Any]] = {}
|
||||
refs: dict[str, str] = {}
|
||||
for side, what in (("base", "基期总数"), ("target", "本期总数")):
|
||||
snap, row, col, value = env.inputs.cell(spec[side], f"headline_{side}", what)
|
||||
if value is None:
|
||||
raise CallError("headline_empty", f"{what}那一格({snap.ref}.r{row}.{col})是空的:换一格有值的")
|
||||
out[side] = (value, snap.cell(row, col))
|
||||
refs[side] = f"{snap.ref}.r{row}.{col}"
|
||||
return refs, out
|
||||
|
||||
|
||||
def _measure_guard(env: Env, snap: Snap, column: str, caveats: list[dict[str, str]], tail: str) -> None:
|
||||
"""additive 的度量列:目录标成(确认过的)比率、标识、状态、属性是调用错误;没标的、只是推断的照算,写附带说明。"""
|
||||
kind, confirmed = env.facts.measure(snap, column)
|
||||
if kind in _NOT_ADDITIVE and confirmed:
|
||||
if kind == "ratio":
|
||||
raise CallError("ratio_as_additive", f"「{column}」是比率(或平均值),不能按段相加:改用比率拆解,分子、分母各给一列")
|
||||
raise CallError("not_a_measure", f"「{column}」在目录里是{_NOT_ADDITIVE[kind]},不是能相加的量:度量列要选金额、数量这类列")
|
||||
if kind is None:
|
||||
caveats.append(ph.caveat("measure_unlabeled", ph.unlabeled_text(column, tail)))
|
||||
elif kind in _NOT_ADDITIVE:
|
||||
caveats.append(ph.caveat("measure_unlabeled", ph.unconfirmed_text(column, kind, tail)))
|
||||
|
||||
|
||||
def _comparability(env: Env, params: dict[str, Any], base: Snap, target: Snap, measure: str | None
|
||||
) -> tuple[cmp.Outcome | None, str | None]:
|
||||
"""两份快照时先做可比性:(结论, 口径结构不同的原因)。"""
|
||||
if "base_query" not in params:
|
||||
return None, None
|
||||
params["today"] = today_param(env)
|
||||
outcome = cmp.compare_queries(base, target, measure, env.facts, env.book, env.today) # type: ignore[arg-type]
|
||||
return outcome, outcome.structure
|
||||
|
||||
|
||||
def _share_rank(segments: Sequence[Any], share: Any) -> list[Any]:
|
||||
"""按 share 从大到小(一样大的按先出现的)。"""
|
||||
return sorted(segments, key=lambda s: -share(s))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# additive
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def additive(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["main_share", "concentration", "offset_ratio", "flat_pct", "table_top_k"]
|
||||
params, bs, ts, cols, by = _sides(raw, env, [("measure", "度量列")])
|
||||
measure = cols["measure"]
|
||||
label = label_for(env, raw.get("label"), ts, measure)
|
||||
if raw.get("label") is not None:
|
||||
params["label"] = ph.clean_label(raw.get("label")) or None
|
||||
dims = [column_label(env, ts, c) for c in by]
|
||||
caveats: list[dict[str, str]] = []
|
||||
_measure_guard(env, ts, measure, caveats, "按可以按段相加处理")
|
||||
refs, heads = _headline(raw, env)
|
||||
if refs:
|
||||
params["headline"] = refs
|
||||
parent_row = None
|
||||
if raw.get("parent") is not None:
|
||||
params["parent"], parent_row, params["depth"] = _parent(raw, env, "additive", len(by))
|
||||
used.append("max_drill_depth")
|
||||
summed = not heads and parent_row is None
|
||||
criterion_args = {"summed": summed}
|
||||
for snap in dict.fromkeys([bs, ts]):
|
||||
if snap.truncated:
|
||||
env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [*by, measure])
|
||||
return refuse(env, params, used, "truncated",
|
||||
f"{snap.ref} 只取回了前 {len(snap.rows)} 行,各段之和不是完整的和",
|
||||
criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args))
|
||||
rows_b, rows_t = _rows_of(raw, params, bs, ts)
|
||||
period_label = column_label(env, ts, params["period"]) if "period" in params else None
|
||||
side_b = _collect(bs, rows_b, by, lambda r: bs.number(r, measure, "度量列"), dims, period_label)
|
||||
side_t = _collect(ts, rows_t, by, lambda r: ts.number(r, measure, "度量列"), dims, period_label)
|
||||
if bs is ts:
|
||||
env.inputs.use(ts, "data", [*by, params["period"], measure], side_b.rows + side_t.rows)
|
||||
else:
|
||||
env.inputs.use(bs, "base", [*by, measure], side_b.rows)
|
||||
env.inputs.use(ts, "target", [*by, measure], side_t.rows)
|
||||
outcome, structure = _comparability(env, params, bs, ts, measure)
|
||||
if structure is not None:
|
||||
return refuse(env, params, used, "incomparable_sides", f"{structure},不是同一个量的两期,不拆",
|
||||
criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats)
|
||||
segs = _segments(side_b, side_t, Fraction(0))
|
||||
if len(segs) < 2:
|
||||
return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(segs)} 段,至少要 2 段才拆得开",
|
||||
criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats)
|
||||
return _additive_result(env, params, used, segs, by, dims, label, heads, parent_row, outcome, caveats, summed,
|
||||
side_b, side_t)
|
||||
|
||||
|
||||
def _additive_result(env: Env, params: dict[str, Any], used: list[str], segs: list[Segment], by: list[str],
|
||||
dims: list[str], label: str, heads: Mapping[str, tuple[Fraction, Any]],
|
||||
parent_row: Mapping[str, Any] | None, outcome: cmp.Outcome | None,
|
||||
caveats: list[dict[str, str]], summed: bool, side_b: Side, side_t: Side) -> Any:
|
||||
s = env.settings
|
||||
main_share, concentration = env.threshold("main_share"), env.threshold("concentration")
|
||||
offset_ratio, flat_pct = env.threshold("offset_ratio"), env.threshold("flat_pct")
|
||||
B = sum((x.base for x in segs), Fraction(0))
|
||||
T = sum((x.target for x in segs), Fraction(0))
|
||||
delta = T - B
|
||||
gaps: dict[str, Fraction] = {}
|
||||
for side, total in (("base", B), ("target", T)):
|
||||
if side in heads and not closes(total, heads[side][0], heads[side][1]):
|
||||
gaps[side] = heads[side][0] - total
|
||||
if parent_row is not None:
|
||||
expect = to_fraction(parent_row.get(side))
|
||||
if expect is None or not closes(total, expect, parent_row.get(side)):
|
||||
gaps.setdefault(side, (expect or Fraction(0)) - total)
|
||||
closed = not gaps
|
||||
deltas = {x.key: x.target - x.base for x in segs}
|
||||
shares = {k: d / delta for k, d in deltas.items()} if closed and delta != 0 else {}
|
||||
gross_up = sum((d for d in deltas.values() if d > 0), Fraction(0))
|
||||
gross_down = sum((d for d in deltas.values() if d < 0), Fraction(0))
|
||||
opposite = abs(gross_down) if delta > 0 else gross_up if delta < 0 else gross_up
|
||||
n = len(segs)
|
||||
k_max = max(2, math.ceil(n / 5))
|
||||
flat = delta == 0 or (B != 0 and abs(delta) / abs(B) < flat_pct)
|
||||
ranked = _share_rank(segs, lambda x: shares[x.key]) if shares else []
|
||||
top_k = top_k_share = None
|
||||
if ranked:
|
||||
running = Fraction(0)
|
||||
for k, seg in enumerate(ranked, start=1):
|
||||
running += shares[seg.key]
|
||||
if running >= concentration:
|
||||
top_k, top_k_share = k, running
|
||||
break
|
||||
if not closed:
|
||||
code, subjects = "unclosed", []
|
||||
elif flat:
|
||||
code, subjects = "flat", []
|
||||
elif shares[ranked[0].key] >= main_share:
|
||||
code, subjects = "driver", [ranked[0]]
|
||||
elif top_k is not None and 2 <= top_k <= k_max:
|
||||
code, subjects = "concentrated", ranked[:top_k]
|
||||
else:
|
||||
code, subjects = "dispersed", []
|
||||
offset = opposite > 0 and opposite >= offset_ratio * abs(delta)
|
||||
if offset:
|
||||
caveats.append(ph.caveat("offset", ph.offset_text(delta, gross_up, gross_down)))
|
||||
nulls = [x for x in segs if "为空" in x.flags]
|
||||
if nulls:
|
||||
caveats.append(ph.caveat("null_values", ph.null_segments_text([ph.key_text(x.key) for x in nulls], label)))
|
||||
if summed:
|
||||
caveats.append(ph.caveat("headline_basis", ph.HEADLINE_BASIS))
|
||||
if outcome is not None and outcome.verdict != "comparable":
|
||||
caveats.append(ph.caveat("comparability", outcome.phrase))
|
||||
top = ranked[0] if ranked else max(segs, key=lambda x: abs(deltas[x.key]))
|
||||
total_text = ph.change(delta, B)
|
||||
top_n = int(s["table_top_k"])
|
||||
if code == "driver":
|
||||
main = ph.decompose_driver(label, total_text, ph.key_text(top.key), ph.amount(deltas[top.key], signed=True),
|
||||
shares[top.key])
|
||||
elif code == "concentrated":
|
||||
main = ph.decompose_concentrated(label, total_text, ph.names([ph.key_text(x.key) for x in subjects], top_n),
|
||||
top_k_share)
|
||||
elif code == "dispersed":
|
||||
main = ph.decompose_dispersed(label, total_text, ph.dims_text(dims), main_share)
|
||||
elif code == "flat":
|
||||
ups = [x for x in segs if deltas[x.key] > 0]
|
||||
downs = [x for x in segs if deltas[x.key] < 0]
|
||||
pair = None
|
||||
if offset and ups and downs:
|
||||
up = max(ups, key=lambda x: deltas[x.key])
|
||||
down = min(downs, key=lambda x: deltas[x.key])
|
||||
pair = (ph.key_text(up.key), ph.amount(deltas[up.key], signed=True), ph.key_text(down.key),
|
||||
ph.amount(deltas[down.key], signed=True))
|
||||
main = ph.decompose_flat(label, delta / abs(B) if B != 0 else None, pair)
|
||||
else:
|
||||
gap = max(gaps.values(), key=abs)
|
||||
main = ph.decompose_unclosed(ph.dims_text(dims), gap, ratio=False)
|
||||
order = sorted(segs, key=lambda x: -abs(deltas[x.key]))
|
||||
rows = []
|
||||
for x in order:
|
||||
d = deltas[x.key]
|
||||
rows.append([*x.key, canon(x.base), canon(x.target), canon(d), canon(d / x.base) if x.base != 0 else None,
|
||||
canon(shares[x.key]) if shares else None, "、".join(x.flags) or None])
|
||||
table = {"columns": [*by, *_FIXED_ADDITIVE],
|
||||
"labels": {**dict(zip(by, dims)), "base": "基期", "target": "本期", "delta": "变化", "pct": "变化率",
|
||||
"share": "占变化的比重", "flag": "说明"},
|
||||
"rows": rows}
|
||||
values: dict[str, Any] = {
|
||||
"base_total": canon(B), "target_total": canon(T), "delta_total": canon(delta),
|
||||
"pct_total": canon(delta / abs(B)) if B != 0 else None, "top_key": ph.key_text(top.key),
|
||||
"top_share": canon(shares[top.key]) if shares else None, "top_k": str(top_k) if top_k is not None else None,
|
||||
"top_k_share": canon(top_k_share) if top_k_share is not None else None, "gross_up": canon(gross_up),
|
||||
"gross_down": canon(gross_down), "segments": str(n),
|
||||
}
|
||||
if gaps:
|
||||
values["gap"] = canon(max(gaps.values(), key=abs))
|
||||
verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects],
|
||||
"direction": sign_word(delta) if code in ("driver", "concentrated", "dispersed") else None}
|
||||
phrase = ph.with_caveats(main, caveats, skip=("offset",) if code == "flat" else ())
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict,
|
||||
criterion=ph.criterion_additive(s, top_k_max=k_max, summed=summed), phrase=phrase, caveats=caveats)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# ratio
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def ratio(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["main_share", "flat_pct", "min_denominator", "table_top_k"]
|
||||
params, bs, ts, cols, by = _sides(raw, env, [("numerator", "分子列"), ("denominator", "分母列")])
|
||||
num, den = cols["numerator"], cols["denominator"]
|
||||
num_label, den_label = column_label(env, ts, num), column_label(env, ts, den)
|
||||
label = ph.pick_label(raw.get("label"), ph.say(num_label, "与", den_label, "之比"))
|
||||
if raw.get("label") is not None:
|
||||
params["label"] = ph.clean_label(raw.get("label")) or None
|
||||
dims = [column_label(env, ts, c) for c in by]
|
||||
refs, heads = _headline(raw, env)
|
||||
if refs:
|
||||
params["headline"] = refs
|
||||
parent_row = None
|
||||
if raw.get("parent") is not None:
|
||||
params["parent"], parent_row, params["depth"] = _parent(raw, env, "ratio", len(by))
|
||||
used.append("max_drill_depth")
|
||||
criterion = ph.criterion_ratio(env.settings)
|
||||
for snap in dict.fromkeys([bs, ts]):
|
||||
if snap.truncated:
|
||||
env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [*by, num, den])
|
||||
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行,各段合起来不是完整的数",
|
||||
criterion=criterion)
|
||||
rows_b, rows_t = _rows_of(raw, params, bs, ts)
|
||||
period_label = column_label(env, ts, params["period"]) if "period" in params else None
|
||||
|
||||
def reader(snap: Snap) -> Any:
|
||||
return lambda r: (snap.number(r, num, "分子列"), snap.number(r, den, "分母列"))
|
||||
|
||||
side_b = _collect(bs, rows_b, by, reader(bs), dims, period_label)
|
||||
side_t = _collect(ts, rows_t, by, reader(ts), dims, period_label)
|
||||
for side in (side_b, side_t):
|
||||
side.nulls = [k for k, v in side.values.items() if v[0] is None or v[1] is None]
|
||||
side.values = {k: (v[0] or Fraction(0), v[1] or Fraction(0)) for k, v in side.values.items()}
|
||||
if bs is ts:
|
||||
env.inputs.use(ts, "data", [*by, params["period"], num, den], side_b.rows + side_t.rows)
|
||||
else:
|
||||
env.inputs.use(bs, "base", [*by, num, den], side_b.rows)
|
||||
env.inputs.use(ts, "target", [*by, num, den], side_t.rows)
|
||||
outcome, structure = _comparability(env, params, bs, ts, den)
|
||||
if structure is not None:
|
||||
return refuse(env, params, used, "incomparable_sides", f"{structure},不是同一个量的两期,不拆", criterion=criterion)
|
||||
segs = _segments(side_b, side_t, (Fraction(0), Fraction(0)))
|
||||
live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0]
|
||||
if len(live) < 2:
|
||||
return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(live)} 段,至少要 2 段才拆得开",
|
||||
criterion=criterion)
|
||||
for when, i in (("基期", 0), ("本期", 1)):
|
||||
if sum(((x.base, x.target)[i][1] for x in live), Fraction(0)) == 0:
|
||||
return refuse(env, params, used, "zero_base", ph.say(f"{when}的", den_label, "合计为 0,算不了比率"),
|
||||
criterion=criterion)
|
||||
for x in live:
|
||||
for (n_, d_), when in ((x.base, "基期"), (x.target, "本期")):
|
||||
if d_ == 0 and n_ != 0:
|
||||
return refuse(env, params, used, "zero_base",
|
||||
ph.say(ph.key_text(x.key), f"{when}的", den_label, "为 0 而", num_label,
|
||||
"不为 0,比率拆不了"), criterion=criterion)
|
||||
return _ratio_result(env, params, used, segs, by, dims, label, num_label, den_label, heads, parent_row, outcome,
|
||||
criterion)
|
||||
|
||||
|
||||
def ratio_effects(segs: Sequence[tuple[Fraction, Fraction, Fraction, Fraction]]
|
||||
) -> tuple[Fraction, Fraction, list[tuple[Fraction, ...]]]:
|
||||
"""比率拆解的计算(细案 §2.2,中点对称拆法)。segs 是各段 (分子_基期, 分母_基期, 分子_本期, 分母_本期),两期分母都为 0
|
||||
的段先剔掉。交回 (R, R', 各段 (w, w', r, r', mix, rate)):Σmix + Σrate = R' − R 逐位成立(Σ(w' − w) = 0)。一边分母为 0
|
||||
的段:那一边权重 0、比率取另一边的(自身效应为 0,变化全记在结构上)。"""
|
||||
db = sum((s[1] for s in segs), Fraction(0))
|
||||
dt = sum((s[3] for s in segs), Fraction(0))
|
||||
R = sum((s[0] for s in segs), Fraction(0)) / db
|
||||
R2 = sum((s[2] for s in segs), Fraction(0)) / dt
|
||||
mid = (R + R2) / 2
|
||||
out = []
|
||||
for nb, d_b, nt, d_t in segs:
|
||||
w, w2 = d_b / db, d_t / dt
|
||||
r = nb / d_b if d_b != 0 else None
|
||||
r2 = nt / d_t if d_t != 0 else None
|
||||
r = r2 if r is None else r
|
||||
r2 = r if r2 is None else r2
|
||||
mix = (w2 - w) * ((r + r2) / 2 - mid) # type: ignore[operator]
|
||||
rate = (r2 - r) * (w + w2) / 2 # type: ignore[operator]
|
||||
out.append((w, w2, r, r2, mix, rate))
|
||||
return R, R2, out
|
||||
|
||||
|
||||
def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[Segment], by: list[str],
|
||||
dims: list[str], label: str, num_label: str, den_label: str,
|
||||
heads: Mapping[str, tuple[Fraction, Any]], parent_row: Mapping[str, Any] | None,
|
||||
outcome: cmp.Outcome | None, criterion: str) -> Any:
|
||||
main_share, flat_pct = env.threshold("main_share"), env.threshold("flat_pct")
|
||||
min_den = env.threshold("min_denominator")
|
||||
live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0]
|
||||
dropped = [x for x in segs if x not in live]
|
||||
R, R2, effects = ratio_effects([(x.base[0], x.base[1], x.target[0], x.target[1]) for x in live])
|
||||
delta = R2 - R
|
||||
by_key = {x.key: e for x, e in zip(live, effects)}
|
||||
gaps: dict[str, Fraction] = {}
|
||||
for side, total in (("base", R), ("target", R2)):
|
||||
if side in heads and not closes(total, heads[side][0], heads[side][1]):
|
||||
gaps[side] = heads[side][0] - total
|
||||
if parent_row is not None:
|
||||
sums = {"n_base": sum((x.base[0] for x in live), Fraction(0)), "d_base": sum((x.base[1] for x in live), Fraction(0)),
|
||||
"n_target": sum((x.target[0] for x in live), Fraction(0)),
|
||||
"d_target": sum((x.target[1] for x in live), Fraction(0))}
|
||||
for col, total in sums.items():
|
||||
expect = to_fraction(parent_row.get(col))
|
||||
if expect is None or not closes(total, expect, parent_row.get(col)):
|
||||
gaps.setdefault("base" if col.endswith("base") else "target", (expect or Fraction(0)) - total)
|
||||
closed = not gaps
|
||||
mix_total = sum((e[4] for e in effects), Fraction(0))
|
||||
rate_total = sum((e[5] for e in effects), Fraction(0))
|
||||
flat = delta == 0 or (R != 0 and abs(delta) / abs(R) < flat_pct)
|
||||
mix_share = mix_total / delta if delta != 0 else None
|
||||
rate_share = rate_total / delta if delta != 0 else None
|
||||
sign = 1 if delta >= 0 else -1
|
||||
if not closed:
|
||||
code, subjects = "unclosed", []
|
||||
elif flat:
|
||||
code, subjects = "flat", []
|
||||
elif mix_share >= main_share and rate_share < main_share: # type: ignore[operator]
|
||||
code = "mix_driven"
|
||||
subjects = [max(live, key=lambda x: sign * by_key[x.key][4])]
|
||||
elif rate_share >= main_share and mix_share < main_share: # type: ignore[operator]
|
||||
code = "rate_driven"
|
||||
subjects = [max(live, key=lambda x: sign * by_key[x.key][5])]
|
||||
else:
|
||||
code, subjects = "both", []
|
||||
proportion = all(0 <= v[0] <= v[1] for x in live for v in (x.base, x.target))
|
||||
small = [x for x in live if x.base[1] < min_den or x.target[1] < min_den]
|
||||
caveats: list[dict[str, str]] = []
|
||||
eligible = [x for x in live if x not in small]
|
||||
if delta != 0 and eligible:
|
||||
weight = sum(((by_key[x.key][0] + by_key[x.key][1]) / 2 for x in eligible), Fraction(0))
|
||||
against = sum(((by_key[x.key][0] + by_key[x.key][1]) / 2 for x in eligible
|
||||
if (by_key[x.key][3] - by_key[x.key][2]) * sign < 0), Fraction(0))
|
||||
if weight > 0 and against / weight >= SIMPSON_SHARE:
|
||||
star = max(live, key=lambda x: sign * by_key[x.key][4])
|
||||
caveats.append(ph.caveat("simpson", ph.simpson_text(label, delta > 0, ph.key_text(star.key),
|
||||
by_key[star.key][0], by_key[star.key][1])))
|
||||
if small:
|
||||
caveats.append(ph.caveat("small_denominator", ph.small_denominator_text(
|
||||
[ph.key_text(x.key) for x in small], den_label, int(env.settings["min_denominator"]))))
|
||||
nulls = [x for x in segs if "为空" in x.flags]
|
||||
if nulls:
|
||||
caveats.append(ph.caveat("null_values", ph.null_segments_text([ph.key_text(x.key) for x in nulls],
|
||||
ph.say(num_label, "或", den_label))))
|
||||
if outcome is not None and outcome.verdict != "comparable":
|
||||
caveats.append(ph.caveat("comparability", outcome.phrase))
|
||||
total_text = ph.ratio_change(R, R2, proportion=proportion)
|
||||
if code == "mix_driven":
|
||||
star = subjects[0]
|
||||
main = ph.ratio_mix(label, total_text, ph.key_text(star.key), by_key[star.key][0], by_key[star.key][1])
|
||||
elif code == "rate_driven":
|
||||
main = ph.ratio_rate(label, total_text, ph.key_text(subjects[0].key))
|
||||
elif code == "both":
|
||||
main = ph.ratio_both(label, total_text, mix_share, rate_share)
|
||||
elif code == "flat":
|
||||
main = ph.decompose_flat(label, delta / abs(R) if R != 0 else None, None)
|
||||
else:
|
||||
main = ph.decompose_unclosed(ph.dims_text(dims), max(gaps.values(), key=abs), ratio=True)
|
||||
rows = []
|
||||
for x in sorted(live, key=lambda x: -abs(by_key[x.key][4] + by_key[x.key][5])):
|
||||
w, w2, r, r2, mix, rate = by_key[x.key]
|
||||
flags = list(x.flags)
|
||||
if x.base[1] == 0:
|
||||
flags.append("基期分母为 0")
|
||||
if x.target[1] == 0:
|
||||
flags.append("本期分母为 0")
|
||||
if x in small:
|
||||
flags.append("分母小")
|
||||
rows.append([*x.key, canon(x.base[0]), canon(x.target[0]), canon(x.base[1]), canon(x.target[1]), canon(w),
|
||||
canon(w2), canon(r), canon(r2), canon(mix), canon(rate), canon(mix + rate),
|
||||
"、".join(dict.fromkeys(flags)) or None])
|
||||
for x in dropped:
|
||||
rows.append([*x.key, canon(x.base[0]), canon(x.target[0]), "0", "0", None, None, None, None, None, None, None,
|
||||
"两期分母都为 0,不计入"])
|
||||
table = {"columns": [*by, *_FIXED_RATIO],
|
||||
"labels": {**dict(zip(by, dims)), "n_base": ph.say("基期", num_label), "n_target": ph.say("本期", num_label),
|
||||
"d_base": ph.say("基期", den_label), "d_target": ph.say("本期", den_label),
|
||||
"w_base": "基期权重", "w_target": "本期权重", "r_base": "基期比率", "r_target": "本期比率",
|
||||
"mix": "结构效应", "rate": "自身效应", "total": "合计影响", "flag": "说明"},
|
||||
"rows": rows}
|
||||
values: dict[str, Any] = {
|
||||
"r_base_total": canon(R), "r_target_total": canon(R2), "delta": canon(delta), "mix_total": canon(mix_total),
|
||||
"rate_total": canon(rate_total), "mix_share": canon(mix_share), "rate_share": canon(rate_share),
|
||||
"segments": str(len(live)),
|
||||
}
|
||||
if gaps:
|
||||
values["gap"] = canon(max(gaps.values(), key=abs))
|
||||
verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects],
|
||||
"direction": sign_word(delta) if code in ("mix_driven", "rate_driven", "both") else None}
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
|
||||
phrase=ph.with_caveats(main, caveats), caveats=caveats)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# factor
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _half_ulp_ratio(raw: Any, value: Fraction) -> Fraction:
|
||||
from app.analysis.common import written_places
|
||||
|
||||
return Fraction(1, 2) / Fraction(10) ** written_places(raw) / abs(value) if value != 0 else Fraction(0)
|
||||
|
||||
|
||||
def lmdi(base: Sequence[float], target: Sequence[float], m_base: float, m_target: float) -> list[float]:
|
||||
"""LMDI-I:contrib_k = L(M', M) · ln(F_k' / F_k),L(a, b) = (a − b) / (ln a − ln b),L(a, a) = a。因子按书写精度取过整、
|
||||
乘积和乘积列差一点时,各因子按同一比例缩放到乘积恰好等于乘积列(每个因子的对数比各补上 (1/K)·差的对数),Σcontrib 在
|
||||
浮点误差内等于 M' − M。"""
|
||||
k = len(base)
|
||||
fix = (math.log(m_target / math.prod(target)) - math.log(m_base / math.prod(base))) / k
|
||||
weight = m_base if m_base == m_target else (m_target - m_base) / (math.log(m_target) - math.log(m_base))
|
||||
return [weight * (math.log(t / b) + fix) for b, t in zip(base, target)]
|
||||
|
||||
|
||||
def factor(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["main_share", "flat_pct"]
|
||||
params, bs, ts, cols, _ = _sides(raw, env, [("product", "乘积列")], by_needed=False)
|
||||
product = cols["product"]
|
||||
wanted = as_list(raw.get("factors"))
|
||||
if not 2 <= len(wanted) <= 4:
|
||||
raise CallError("bad_factors", "因子列要写 2 到 4 列")
|
||||
factors = [ts.column(c, "因子列") for c in wanted]
|
||||
if bs is not ts:
|
||||
for c in factors:
|
||||
bs.column(c, "因子列")
|
||||
if len(set(factors)) != len(factors) or product in factors:
|
||||
raise CallError("bad_factors", "因子列不能重复,也不能是乘积列本身")
|
||||
params["factors"] = factors
|
||||
label = label_for(env, raw.get("label"), ts, product)
|
||||
if raw.get("label") is not None:
|
||||
params["label"] = ph.clean_label(raw.get("label")) or None
|
||||
names = [column_label(env, ts, c) for c in factors]
|
||||
criterion = ph.criterion_factor(env.settings)
|
||||
for snap in dict.fromkeys([bs, ts]):
|
||||
if snap.truncated:
|
||||
env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [product, *factors])
|
||||
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行,结果不完整",
|
||||
criterion=criterion)
|
||||
if "period" in params:
|
||||
rows_b = _period_rows(bs, params["period"], params["base"], "基期")
|
||||
rows_t = _period_rows(ts, params["period"], params["target"], "本期")
|
||||
else:
|
||||
rows_b, rows_t = list(range(len(bs.rows))), list(range(len(ts.rows)))
|
||||
for rows, snap, when in ((rows_b, bs, "基期"), (rows_t, ts, "本期")):
|
||||
if len(rows) != 1:
|
||||
raise CallError("duplicate_rows", f"{snap.ref} 里{when}有 {len(rows)} 行:乘积拆解每期要正好一行(在 SQL 里汇总成一行)")
|
||||
cells: dict[str, list[tuple[Fraction, Any]]] = {}
|
||||
for when, snap, r in (("base", bs, rows_b[0]), ("target", ts, rows_t[0])):
|
||||
row = []
|
||||
for col, what in ((product, "乘积列"), *((c, "因子列") for c in factors)):
|
||||
value = snap.number(r, col, what)
|
||||
if value is None:
|
||||
raise CallError("empty_cell", f"{snap.ref} 第 {r} 行的「{col}」是空的:乘积、因子都要有值")
|
||||
row.append((value, snap.cell(r, col)))
|
||||
cells[when] = row
|
||||
if bs is ts:
|
||||
env.inputs.use(ts, "data", [params["period"], product, *factors], rows_b + rows_t)
|
||||
else:
|
||||
env.inputs.use(bs, "base", [product, *factors], rows_b)
|
||||
env.inputs.use(ts, "target", [product, *factors], rows_t)
|
||||
outcome, structure = _comparability(env, params, bs, ts, product)
|
||||
if structure is not None:
|
||||
return refuse(env, params, used, "incomparable_sides", f"{structure},不是同一个量的两期,不拆", criterion=criterion)
|
||||
if any(v <= 0 for row in cells.values() for v, _ in row):
|
||||
return refuse(env, params, used, "non_positive", "乘积或因子里有不大于 0 的值,对数拆解要求全为正", criterion=criterion)
|
||||
for when, row in cells.items():
|
||||
m = row[0][0]
|
||||
prod = math.prod((v for v, _ in row[1:]), start=Fraction(1))
|
||||
rel = abs(m - prod) / abs(m)
|
||||
allowed = max(_FACTOR_REL, sum((_half_ulp_ratio(raw_v, v) for v, raw_v in row[1:]), Fraction(0)))
|
||||
if rel > allowed:
|
||||
raise CallError("not_product", ph.say("这几列相乘不等于", column_label(env, ts, product),
|
||||
f"({'基期' if when == 'base' else '本期'}差 {ph.pct(rel, signed=False)}),"
|
||||
"不是乘积关系:换成乘起来正好等于它的几列"))
|
||||
M, M2 = cells["base"][0][0], cells["target"][0][0]
|
||||
fb = [float(v) for v, _ in cells["base"][1:]]
|
||||
ft = [float(v) for v, _ in cells["target"][1:]]
|
||||
contrib = lmdi(fb, ft, float(M), float(M2))
|
||||
delta = M2 - M
|
||||
flat = delta == 0 or abs(delta) / abs(M) < env.threshold("flat_pct")
|
||||
shares = [c / float(delta) for c in contrib] if delta != 0 else [None] * len(contrib)
|
||||
main_share = float(env.threshold("main_share"))
|
||||
order = sorted(range(len(factors)), key=lambda i: -abs(contrib[i]))
|
||||
caveats: list[dict[str, str]] = []
|
||||
if outcome is not None and outcome.verdict != "comparable":
|
||||
caveats.append(ph.caveat("comparability", outcome.phrase))
|
||||
total_text = ph.change(delta, M)
|
||||
top = max(range(len(factors)), key=lambda i: shares[i] if shares[i] is not None else -math.inf)
|
||||
if flat:
|
||||
code, subjects = "flat", []
|
||||
main = ph.decompose_flat(label, delta / abs(M), None)
|
||||
elif shares[top] >= main_share: # type: ignore[operator]
|
||||
code, subjects = "driver", [[names[top]]]
|
||||
main = ph.factor_driver(label, total_text, names[top], contrib[top], shares[top])
|
||||
else:
|
||||
code, subjects = "both", []
|
||||
main = ph.factor_both(label, total_text, [(names[i], shares[i]) for i in order])
|
||||
rows = []
|
||||
for i in order:
|
||||
b, t = cells["base"][i + 1][0], cells["target"][i + 1][0]
|
||||
rows.append([names[i], canon(b), canon(t), canon((t - b) / b), canon(contrib[i]),
|
||||
canon(shares[i]) if shares[i] is not None else None])
|
||||
table = {"columns": ["factor", "base", "target", "pct", "contrib", "share"],
|
||||
"labels": {"factor": "因子", "base": "基期", "target": "本期", "pct": "变化率", "contrib": "贡献",
|
||||
"share": "占变化的比重"},
|
||||
"rows": rows}
|
||||
values = {"base_total": canon(M), "target_total": canon(M2), "delta_total": canon(delta),
|
||||
"pct_total": canon(delta / abs(M)), "top_key": names[top],
|
||||
"top_share": canon(shares[top]) if shares[top] is not None else None}
|
||||
verdict = {"code": code, "subjects": subjects, "direction": sign_word(delta) if code != "flat" else None}
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
|
||||
phrase=ph.with_caveats(main, caveats), caveats=caveats)
|
||||
|
||||
|
||||
__all__ = ["HEADLINE_SHAPE", "PARENT_SHAPE", "SAME_QUERY", "SIMPSON_SHARE", "additive", "factor", "lmdi", "ratio", "ratio_effects"]
|
||||
@@ -0,0 +1,371 @@
|
||||
"""差异检验(细案 01 §5):这两组(或两期)真的不一样,还是样本波动。
|
||||
|
||||
**参数**:query(一份快照);按组比时 by(分组列,一列);按期比同一组时 period、base、target(各一行,SQL 里先筛好那一组)。
|
||||
比哪几对:pairs([[A, B], …],A、B 是分组列里的值)、reference(对照组:其余各组各和它比);都不写时两组就比这两组,三组
|
||||
以上每组和「其余各组合计」比(和含它自己的整体比不独立)。可选 label。
|
||||
|
||||
- rate:numerator(成功数,如转化单数)、denominator(试验数,如访客数),每组一行,都要是不小于 0 的整数、成功数不超过
|
||||
试验数。每组 Wilson 区间,两组之差 Newcombe 区间;最小期望频数不小于 min_expected 用合并比例的 z 检验,否则用 Fisher
|
||||
精确检验。
|
||||
- mean:每组一行,mean、sd、n 三列(均值、标准差、样本数),或者 sum、sumsq、n 三列(合计、平方和、样本数:SQLite 没有标准差
|
||||
函数,SUM(x)、SUM(x*x)、COUNT(*) 都写得出)。Welch t 检验,自由度按 Welch–Satterthwaite。
|
||||
- rows:value(逐行的值)+ 分组列(或期次列),快照不超过 1000 行、没截断;算子自己算均值、方差,另做敏感性:去掉离均值
|
||||
最远的那一个值后重算,判定变了加 outlier_sensitive。
|
||||
|
||||
比了几对时按 Holm 校正,表里同时给原始 p 和校正后的 p,附 multiple_comparisons。校正后 p < differ_alpha → differs(direction
|
||||
按 A − B 的正负);否则 no_evidence。用了 Fisher(比率)或有一组样本数小于 min_n(均值)→ small_sample。判定语不出现「p 值」
|
||||
「显著性」:differs 说「差异站得住」,no_evidence 说「差异可能只是样本波动」,p 值、区间在表和证据面板里。
|
||||
|
||||
subjects 是判定说的那一对 [[A], [B]](只比一对时就是那一对,比了几对时是校正后最站得住的那一对;几对都不站得住时为空),
|
||||
direction 是 A 相对 B。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis import stats
|
||||
from app.analysis.common import Env, build, column_label, label_for, need, refuse, sign_word
|
||||
from app.analysis.contract import CallError
|
||||
from app.analysis.inputs import Snap, as_list, hashable, same_value
|
||||
from app.analysis.numbers import canon, to_fraction
|
||||
|
||||
TESTS = {"z": "z 检验", "fisher": "Fisher 精确检验", "welch": "Welch t 检验"}
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class Group:
|
||||
key: Any
|
||||
name: str
|
||||
rows: list[int] = field(default_factory=list)
|
||||
x: int = 0 # rate:成功数
|
||||
n: int = 0 # rate:试验数;mean / rows:样本数
|
||||
mean: Fraction | None = None
|
||||
var: Fraction | None = None
|
||||
values: list[Fraction] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Pair:
|
||||
a: Group
|
||||
b: Group
|
||||
test: str | None = None
|
||||
p: float | None = None
|
||||
p_adj: float | None = None
|
||||
diff: Fraction | None = None
|
||||
ci: tuple[float, float] | None = None
|
||||
a_value: Fraction | None = None
|
||||
b_value: Fraction | None = None
|
||||
|
||||
|
||||
def _int(value: Fraction | None, snap: Snap, r: int, column: str) -> int:
|
||||
if value is None or value.denominator != 1 or value < 0:
|
||||
raise CallError("not_count", f"{snap.ref} 第 {r} 行的「{column}」不是不小于 0 的整数:比率比较的分子、分母要是计数")
|
||||
return int(value)
|
||||
|
||||
|
||||
def _pool(name: str, groups: Sequence[Group], mode: str) -> Group:
|
||||
"""「其余各组合计」:比率把成功数、试验数加起来;均值按各组的均值、方差、样本数合并;逐行的值并在一起。"""
|
||||
out = Group(("__rest__",), name, rows=[r for g in groups for r in g.rows])
|
||||
if mode == "rate":
|
||||
out.x, out.n = sum(g.x for g in groups), sum(g.n for g in groups)
|
||||
return out
|
||||
if mode == "rows":
|
||||
out.values = [v for g in groups for v in g.values]
|
||||
return _moments(out)
|
||||
out.n = sum(g.n for g in groups)
|
||||
if out.n == 0:
|
||||
return out
|
||||
out.mean = sum((g.mean * g.n for g in groups if g.mean is not None), Fraction(0)) / out.n
|
||||
if out.n > 1:
|
||||
within = sum(((g.n - 1) * g.var for g in groups if g.var is not None and g.n > 1), Fraction(0))
|
||||
between = sum((g.n * (g.mean - out.mean) ** 2 for g in groups if g.mean is not None), Fraction(0))
|
||||
out.var = (within + between) / (out.n - 1)
|
||||
return out
|
||||
|
||||
|
||||
def _moments(group: Group) -> Group:
|
||||
group.n = len(group.values)
|
||||
if group.n:
|
||||
group.mean = sum(group.values, Fraction(0)) / group.n
|
||||
if group.n > 1:
|
||||
group.var = sum(((v - group.mean) ** 2 for v in group.values), Fraction(0)) / (group.n - 1) # type: ignore[operator]
|
||||
return group
|
||||
|
||||
|
||||
def _test(pair: Pair, mode: str, min_expected: Fraction) -> None:
|
||||
a, b = pair.a, pair.b
|
||||
if mode == "rate":
|
||||
if a.n == 0 or b.n == 0:
|
||||
return
|
||||
pair.a_value, pair.b_value = Fraction(a.x, a.n), Fraction(b.x, b.n)
|
||||
pair.diff = pair.a_value - pair.b_value
|
||||
pair.ci = stats.newcombe(a.x, a.n, b.x, b.n)
|
||||
if stats.min_expected(a.x, a.n, b.x, b.n) >= min_expected:
|
||||
pair.test = "z"
|
||||
pair.p = stats.pooled_z(a.x, a.n, b.x, b.n)[1] # type: ignore[index]
|
||||
else:
|
||||
pair.test = "fisher"
|
||||
pair.p = stats.fisher_exact(a.x, a.n - a.x, b.x, b.n - b.x)
|
||||
return
|
||||
if a.mean is None or b.mean is None:
|
||||
return
|
||||
pair.a_value, pair.b_value = a.mean, b.mean
|
||||
pair.diff = a.mean - b.mean
|
||||
if a.var is None or b.var is None:
|
||||
return
|
||||
test = stats.welch(float(a.mean), float(a.var), a.n, float(b.mean), float(b.var), b.n)
|
||||
if test is None:
|
||||
return
|
||||
pair.test, pair.p = "welch", test.p
|
||||
pair.ci = stats.welch_interval(float(a.mean), float(a.var), a.n, float(b.mean), float(b.var), b.n)
|
||||
|
||||
|
||||
def _verdict(pairs: Sequence[Pair], alpha: Fraction) -> list[Pair]:
|
||||
"""Holm 校正后站得住的那几对(按校正后的 p 从小到大,一样的按先后)。"""
|
||||
adjusted = stats.holm([p.p for p in pairs])
|
||||
for pair, adj in zip(pairs, adjusted):
|
||||
pair.p_adj = adj
|
||||
hits = [p for p in pairs if p.p_adj is not None and to_fraction(p.p_adj) < alpha] # type: ignore[operator]
|
||||
return sorted(hits, key=lambda p: p.p_adj) # type: ignore[arg-type, return-value]
|
||||
|
||||
|
||||
def differ(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
mode = env.mode
|
||||
used = ["differ_alpha", "min_expected"] if mode == "rate" else ["differ_alpha", "min_n"]
|
||||
snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
|
||||
params: dict[str, Any] = {"query": snap.ref}
|
||||
by_period = raw.get("period") is not None
|
||||
if by_period:
|
||||
group_col = snap.column(raw.get("period"), "期次列")
|
||||
params.update(period=group_col, base=need(raw, "base", "基期(期次列里的值)"),
|
||||
target=need(raw, "target", "本期(期次列里的值)"))
|
||||
else:
|
||||
wanted = as_list(raw.get("by"))
|
||||
if len(wanted) != 1:
|
||||
raise CallError("bad_by", "差异检验按一列分组:分组列写一列;比同一组的两期就写期次列、基期、本期")
|
||||
group_col = snap.column(wanted[0], "分组列")
|
||||
params["by"] = [group_col]
|
||||
if mode == "rate":
|
||||
cols = {"numerator": snap.column(need(raw, "numerator", "成功数列"), "成功数列"),
|
||||
"denominator": snap.column(need(raw, "denominator", "试验数列"), "试验数列")}
|
||||
elif mode == "mean":
|
||||
if raw.get("mean") is not None:
|
||||
cols = {"mean": snap.column(raw.get("mean"), "均值列"), "sd": snap.column(need(raw, "sd", "标准差列"), "标准差列"),
|
||||
"n": snap.column(need(raw, "n", "样本数列"), "样本数列")}
|
||||
else:
|
||||
cols = {"sum": snap.column(need(raw, "sum", "合计列(或均值列)"), "合计列"),
|
||||
"sumsq": snap.column(need(raw, "sumsq", "平方和列"), "平方和列"),
|
||||
"n": snap.column(need(raw, "n", "样本数列"), "样本数列")}
|
||||
else:
|
||||
cols = {"value": snap.column(need(raw, "value", "值列"), "值列")}
|
||||
params.update(cols)
|
||||
measure_col = cols.get("numerator") or cols.get("mean") or cols.get("sum") or cols.get("value")
|
||||
if mode == "rate":
|
||||
label = ph.pick_label(raw.get("label"), ph.say(column_label(env, snap, cols["numerator"]), "占",
|
||||
column_label(env, snap, cols["denominator"]), "的比率"))
|
||||
else:
|
||||
label = label_for(env, raw.get("label"), snap, measure_col)
|
||||
if raw.get("label") is not None:
|
||||
params["label"] = ph.clean_label(raw.get("label")) or None
|
||||
dim_label = column_label(env, snap, group_col)
|
||||
criterion = ph.criterion_differ(env.settings, mode) # type: ignore[arg-type]
|
||||
if snap.truncated:
|
||||
env.inputs.use(snap, "data", [group_col, *cols.values()])
|
||||
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行,各组不完整",
|
||||
criterion=criterion)
|
||||
groups = _groups(snap, group_col, cols, mode, by_period, params) # type: ignore[arg-type]
|
||||
pairs = _pairs(raw, params, groups, mode, dim_label, by_period, group_col) # type: ignore[arg-type]
|
||||
env.inputs.use(snap, "data", [group_col, *cols.values()],
|
||||
sorted({r for p in pairs for g in (p.a, p.b) for r in g.rows}))
|
||||
return _decide(env, params, used, pairs, mode, label, criterion) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _groups(snap: Snap, group_col: str, cols: Mapping[str, str], mode: str, by_period: bool,
|
||||
params: Mapping[str, Any]) -> dict[Any, Group]:
|
||||
gi = snap.index(group_col)
|
||||
groups: dict[Any, Group] = {}
|
||||
wanted = [params["target"], params["base"]] if by_period else None
|
||||
for r, row in enumerate(snap.rows):
|
||||
value = row[gi] if gi < len(row) else None
|
||||
if wanted is not None:
|
||||
if not any(same_value(w, value) for w in wanted):
|
||||
continue
|
||||
key = hashable(value)
|
||||
group = groups.setdefault(key, Group(key, ph.key_text([value])))
|
||||
if mode != "rows" and group.rows:
|
||||
raise CallError("duplicate_rows", f"{snap.ref} 里「{ph.key_text([value])}」有不止一行:在 SQL 里按 {group_col} "
|
||||
"汇总,每组一行")
|
||||
group.rows.append(r)
|
||||
if mode == "rate":
|
||||
group.x = _int(snap.number(r, cols["numerator"], "成功数列"), snap, r, cols["numerator"])
|
||||
group.n = _int(snap.number(r, cols["denominator"], "试验数列"), snap, r, cols["denominator"])
|
||||
if group.x > group.n:
|
||||
raise CallError("not_count", f"{snap.ref} 第 {r} 行的成功数大于试验数:分子、分母要是同一批里的「成功几个、"
|
||||
"一共几个」")
|
||||
elif mode == "mean":
|
||||
n = _int(snap.number(r, cols["n"], "样本数列"), snap, r, cols["n"])
|
||||
group.n = n
|
||||
if "mean" in cols:
|
||||
group.mean = snap.number(r, cols["mean"], "均值列")
|
||||
sd = snap.number(r, cols["sd"], "标准差列")
|
||||
group.var = sd * sd if sd is not None and n > 1 else None
|
||||
else:
|
||||
total = snap.number(r, cols["sum"], "合计列")
|
||||
sq = snap.number(r, cols["sumsq"], "平方和列")
|
||||
if total is not None and n > 0:
|
||||
group.mean = total / n
|
||||
if sq is not None and n > 1:
|
||||
group.var = max(Fraction(0), (sq - total * total / n) / (n - 1))
|
||||
else:
|
||||
v = snap.number(r, cols["value"], "值列")
|
||||
if v is not None:
|
||||
group.values.append(v)
|
||||
if mode == "rows":
|
||||
for g in groups.values():
|
||||
_moments(g)
|
||||
if by_period:
|
||||
for w, what in ((params["target"], "本期"), (params["base"], "基期")):
|
||||
if not any(same_value(w, g.key) for g in groups.values()):
|
||||
raise CallError("period_not_found", f"期次列「{group_col}」里没有「{w}」:{what}要写 {snap.ref} 里有的期次值")
|
||||
return groups
|
||||
|
||||
|
||||
def _find(groups: Mapping[Any, Group], value: Any, column: str) -> Group:
|
||||
found = next((g for g in groups.values() if same_value(value, g.key)), None)
|
||||
if found is None:
|
||||
shown = "、".join(g.name for g in list(groups.values())[:8])
|
||||
raise CallError("unknown_group", f"「{value}」不是「{column}」里的值:要比的组写 {shown} 里的")
|
||||
return found
|
||||
|
||||
|
||||
def _pairs(raw: Mapping[str, Any], params: dict[str, Any], groups: dict[Any, Group], mode: str, dim_label: str,
|
||||
by_period: bool, group_col: str) -> list[Pair]:
|
||||
if by_period:
|
||||
target = _find(groups, params["target"], group_col)
|
||||
base = _find(groups, params["base"], group_col)
|
||||
return [Pair(target, base)]
|
||||
if raw.get("pairs") is not None:
|
||||
spec = raw.get("pairs")
|
||||
if not isinstance(spec, list) or not spec or not all(isinstance(p, (list, tuple)) and len(p) == 2 for p in spec):
|
||||
raise CallError("bad_pairs", "要比的几对写成 [[A, B], …],A、B 是分组列里的值")
|
||||
out = []
|
||||
for a, b in spec:
|
||||
ga, gb = _find(groups, a, group_col), _find(groups, b, group_col)
|
||||
if ga is gb:
|
||||
raise CallError("bad_pairs", f"「{ga.name}」不能和自己比")
|
||||
out.append(Pair(ga, gb))
|
||||
params["pairs"] = [[p.a.key, p.b.key] for p in out]
|
||||
return out
|
||||
if len(groups) < 2:
|
||||
raise CallError("too_few_groups", f"「{group_col}」里只有 {len(groups)} 组,至少要两组才比得了")
|
||||
if raw.get("reference") is not None:
|
||||
ref = _find(groups, raw.get("reference"), group_col)
|
||||
params["reference"] = ref.key
|
||||
return [Pair(g, ref) for g in groups.values() if g is not ref]
|
||||
items = list(groups.values())
|
||||
if len(items) == 2:
|
||||
return [Pair(items[0], items[1])]
|
||||
rest_name = ph.say("其余各", dim_label)
|
||||
return [Pair(g, _pool(rest_name, [o for o in items if o is not g], mode)) for g in items]
|
||||
|
||||
|
||||
def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair], mode: str, label: str,
|
||||
criterion: str) -> Any:
|
||||
alpha = env.threshold("differ_alpha")
|
||||
min_expected = env.threshold("min_expected") if mode == "rate" else Fraction(0)
|
||||
for pair in pairs:
|
||||
_test(pair, mode, min_expected)
|
||||
hits = _verdict(pairs, alpha)
|
||||
caveats: list[dict[str, str]] = []
|
||||
if mode == "rows":
|
||||
flipped = _sensitivity(pairs, mode, alpha, min_expected, bool(hits))
|
||||
if flipped is not None:
|
||||
caveats.append(ph.caveat("outlier_sensitive", ph.outlier_text(*flipped)))
|
||||
hits = _verdict(pairs, alpha)
|
||||
if mode == "rate" and any(p.test == "fisher" for p in pairs):
|
||||
caveats.append(ph.caveat("small_sample", ph.small_sample_rate_text(int(env.settings["min_expected"]))))
|
||||
if mode != "rate":
|
||||
min_n = int(env.settings["min_n"])
|
||||
small = sorted((g for p in pairs for g in (p.a, p.b) if g.n < min_n), key=lambda g: g.n)
|
||||
if small:
|
||||
caveats.append(ph.caveat("small_sample", ph.small_sample_mean_text(small[0].name, small[0].n, min_n)))
|
||||
tested = [p for p in pairs if p.p is not None]
|
||||
if len(tested) > 1:
|
||||
caveats.append(ph.caveat("multiple_comparisons", ph.multiple_text(len(tested))))
|
||||
fmt = (lambda v: ph.pct(v, signed=False)) if mode == "rate" else (lambda v: ph.amount(v))
|
||||
if hits:
|
||||
code = "differs"
|
||||
head = hits[0]
|
||||
main = ph.differ_phrase([ph.differ_pair(p.a.name, p.b.name, label, fmt(p.a_value), fmt(p.b_value),
|
||||
p.diff > 0) for p in hits], # type: ignore[operator]
|
||||
len(pairs))
|
||||
subjects = [[head.a.name], [head.b.name]]
|
||||
direction = sign_word(head.diff) # type: ignore[arg-type]
|
||||
else:
|
||||
code = "no_evidence"
|
||||
head = pairs[0] if len(pairs) == 1 else None
|
||||
sizes = (ph.amount(head.a.n), ph.amount(head.b.n)) if head is not None else None
|
||||
main = ph.no_evidence_phrase(head.a.name if head else None, head.b.name if head else None, label, sizes,
|
||||
len(pairs))
|
||||
subjects = [[head.a.name], [head.b.name]] if head is not None else []
|
||||
direction = None
|
||||
value_name = "比率" if mode == "rate" else "均值"
|
||||
rows = []
|
||||
for p in pairs:
|
||||
flags = []
|
||||
if p.p is None:
|
||||
flags.append("样本不够,算不了")
|
||||
if p.test == "fisher":
|
||||
flags.append("样本少,用了精确检验")
|
||||
rows.append([p.a.name, p.b.name, canon(p.a_value), canon(p.b_value), canon(p.diff),
|
||||
canon(p.ci[0]) if p.ci else None, canon(p.ci[1]) if p.ci else None, str(p.a.n), str(p.b.n),
|
||||
TESTS.get(p.test or "") or None, canon(p.p), canon(p.p_adj), "、".join(flags) or None])
|
||||
table = {"columns": ["a", "b", "a_value", "b_value", "diff", "ci_lo", "ci_hi", "a_n", "b_n", "test", "p", "p_adj",
|
||||
"flag"],
|
||||
"labels": {"a": "组", "b": "对照组", "a_value": f"组的{value_name}", "b_value": f"对照组的{value_name}",
|
||||
"diff": "差", "ci_lo": "差的区间下限", "ci_hi": "差的区间上限", "a_n": "组的样本量",
|
||||
"b_n": "对照组的样本量", "test": "检验", "p": "p 值", "p_adj": "校正后的 p 值", "flag": "说明"},
|
||||
"rows": rows}
|
||||
shown = head if head is not None else pairs[0]
|
||||
values = {"a": canon(shown.a_value), "b": canon(shown.b_value), "diff": canon(shown.diff),
|
||||
"ci_lo": canon(shown.ci[0]) if shown.ci else None, "ci_hi": canon(shown.ci[1]) if shown.ci else None,
|
||||
"p": canon(shown.p), "p_adj": canon(shown.p_adj), "n_a": str(shown.a.n), "n_b": str(shown.b.n),
|
||||
"test": shown.test, "pairs": str(len(pairs)), "differs": str(len(hits))}
|
||||
verdict = {"code": code, "subjects": subjects, "direction": direction}
|
||||
phrase = ph.with_caveats(main, caveats, skip=("multiple_comparisons",) if len(pairs) > 1 else ())
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, phrase=phrase,
|
||||
caveats=caveats)
|
||||
|
||||
|
||||
def _sensitivity(pairs: list[Pair], mode: str, alpha: Fraction, min_expected: Fraction,
|
||||
differs: bool) -> tuple[bool, bool] | None:
|
||||
"""rows 模式的敏感性:去掉比较的各组里离均值最远的那一个值重算,判定变了交回 (去掉的是不是偏大的那一笔, 现在站不站得住)。"""
|
||||
everything = [(g, i, v) for g in dict.fromkeys(g for p in pairs for g in (p.a, p.b)) for i, v in enumerate(g.values)]
|
||||
if len(everything) < 3:
|
||||
return None
|
||||
mean = sum((v for _, _, v in everything), Fraction(0)) / len(everything)
|
||||
group, index, value = max(everything, key=lambda t: abs(t[2] - mean))
|
||||
trimmed = []
|
||||
for p in pairs:
|
||||
def cut(g: Group) -> Group:
|
||||
if g is not group and not (g.key == ("__rest__",) and any(v is value for v in g.values)):
|
||||
return g
|
||||
values = list(g.values)
|
||||
for k, v in enumerate(values):
|
||||
if v is value:
|
||||
del values[k]
|
||||
break
|
||||
return _moments(Group(g.key, g.name, rows=g.rows, values=values))
|
||||
trimmed.append(Pair(cut(p.a), cut(p.b)))
|
||||
for p in trimmed:
|
||||
_test(p, mode, min_expected)
|
||||
now = bool(_verdict(trimmed, alpha))
|
||||
_verdict(pairs, alpha)
|
||||
return (value > mean, now) if now != differs else None
|
||||
|
||||
|
||||
__all__ = ["TESTS", "differ"]
|
||||
@@ -0,0 +1,218 @@
|
||||
"""目录事实:结果列给人看的叫法、度量类型,表的业务日期(细案 01 §2.1、§6、§7.3)。
|
||||
|
||||
算子要知道「gmv 是流量还是比率」(比率当可加是调用错误;窗口不等时流量判不可比、其余只提醒)、「region 叫地区」(判定语
|
||||
写「分散在各地区」)、「orders 的业务日期是 order_date」(可比性检查按它取时间条件)。这些在数据目录里,可目录描述的是表的
|
||||
列,快照里的是 SQL 的结果列,所以先按 SQL 认结果列取自哪张表的哪一列,认法同单元格的显示规则(engine/cell_format:最外层
|
||||
SELECT 里按别名或列名找到那一项,看它是不是聚合、聚合的参数是不是一列、这一列在 FROM / JOIN 的哪张实表上),再查那一列
|
||||
的目录项:
|
||||
|
||||
- SUM(x)、不聚合的 x:x 的度量类型;COUNT(*) / COUNT(x):计数,按流量算;AVG(x):平均值,按比率算(不能按段相加);
|
||||
COUNT(DISTINCT x)、MIN / MAX、算式:认不出(当没标)。
|
||||
- 叫法:x 的中文名(目录里的 label);没有就是没有,调用方用列名。
|
||||
- 只认没被驳回的项(catalog.visible_notes)。度量类型另记有没有人确认过(confirmed / verified):只有确认过的才拿来判
|
||||
调用错误,推断的只写进附带说明(同 SQL 检查「宁可漏报,不可误报」)。
|
||||
|
||||
**复算不读目录**:run 时查过的都记进结论的 params["catalog"](record),复算时 Facts 从那里读,同一个键得到同一个答案;
|
||||
目录事后改了,封存的结论照样复算得出来。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
from app.data.names import name_key
|
||||
|
||||
#: 有确证的项状态:只有它们拿来判调用错误
|
||||
_STRONG = frozenset({"confirmed", "verified"})
|
||||
|
||||
|
||||
def _notes_of(entry: Any) -> Mapping[str, Any] | None:
|
||||
"""目录里一张表的 notes:CatalogEntry(有 .notes)或 notes 本身;去掉被驳回的项。"""
|
||||
from app.data import catalog
|
||||
|
||||
notes = getattr(entry, "notes", entry)
|
||||
return catalog.visible_notes(notes) if isinstance(notes, Mapping) else None
|
||||
|
||||
|
||||
def _table_notes(catalog: Mapping[str, Any] | None, table: str | None) -> Mapping[str, Any] | None:
|
||||
if not catalog or not table:
|
||||
return None
|
||||
wanted = {name_key(table), name_key(table.rsplit(".", 1)[-1])}
|
||||
for name, entry in catalog.items():
|
||||
if name_key(str(name)) in wanted or name_key(str(name).rsplit(".", 1)[-1]) in wanted:
|
||||
return _notes_of(entry)
|
||||
return None
|
||||
|
||||
|
||||
def _item(notes: Mapping[str, Any] | None, column: str, key: str) -> Mapping[str, Any] | None:
|
||||
columns = (notes or {}).get("columns") if isinstance(notes, Mapping) else None
|
||||
if not isinstance(columns, Mapping):
|
||||
return None
|
||||
for name, items in columns.items():
|
||||
if name_key(str(name)) == name_key(column) and isinstance(items, Mapping):
|
||||
item = items.get(key)
|
||||
return item if isinstance(item, Mapping) else None
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def _outer_select(sql: str) -> Any:
|
||||
"""最外层的 SELECT(engine/cell_format 的读法);同一条 SQL 只解析一次(只读,不改这棵树)。"""
|
||||
from app.engine import cell_format as cf
|
||||
|
||||
try:
|
||||
return cf._outer_select(sql)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
def source_of(sql: str, column: str) -> tuple[str | None, str | None, str | None, bool] | None:
|
||||
"""结果列 column 取自哪里:(表, 列, 聚合, 去重)。聚合是 sum / count / avg / min / max / other(别的聚合)或 None(不聚合);
|
||||
表在 SELECT * 、子查询、认不准的时候为 None。SQL 解析不了、结果列在最外层 SELECT 里找不到为 None。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.engine import cell_format as cf
|
||||
|
||||
try:
|
||||
select = _outer_select(sql)
|
||||
if select is None:
|
||||
return None
|
||||
item = cf._projection(select, column)
|
||||
if item is None:
|
||||
return None
|
||||
value = cf._unwrap(item.unalias())
|
||||
agg: str | None = None
|
||||
distinct = False
|
||||
if isinstance(value, exp.AggFunc):
|
||||
agg = {exp.Sum: "sum", exp.Count: "count", exp.Avg: "avg", exp.Min: "min", exp.Max: "max"}.get(
|
||||
type(value), "other")
|
||||
arg = value.this
|
||||
if isinstance(arg, exp.Distinct):
|
||||
distinct = True
|
||||
arg = arg.expressions[0] if len(arg.expressions) == 1 else None
|
||||
value = cf._unwrap(arg) if arg is not None else None
|
||||
if not isinstance(value, exp.Column) or isinstance(value.this, exp.Star):
|
||||
return None, None, agg, distinct
|
||||
tables = cf._tables(select)
|
||||
table = tables.get(name_key(value.table)) if value.table else \
|
||||
(next(iter(tables.values())) if len(tables) == 1 else None)
|
||||
return table, value.name, agg, distinct
|
||||
except Exception: # noqa: BLE001 - 认不出就当没标
|
||||
return None
|
||||
|
||||
|
||||
def date_source(sql: str, column: str) -> tuple[str, str] | None:
|
||||
"""期次结果列取自哪张表的哪个日期列:那一项选取里只引用了一列(order_date、DATE(order_date)、
|
||||
strftime('%Y-%m', order_date))时就是它。认不出为 None。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.engine import cell_format as cf
|
||||
|
||||
try:
|
||||
select = _outer_select(sql)
|
||||
item = cf._projection(select, column) if select is not None else None
|
||||
if item is None:
|
||||
return None
|
||||
cols = [c for c in item.unalias().find_all(exp.Column) if not isinstance(c.this, exp.Star)]
|
||||
if len({(name_key(c.table), name_key(c.name)) for c in cols}) != 1:
|
||||
return None
|
||||
ref = cols[0]
|
||||
tables = cf._tables(select)
|
||||
table = tables.get(name_key(ref.table)) if ref.table else \
|
||||
(next(iter(tables.values())) if len(tables) == 1 else None)
|
||||
return (table, ref.name) if table else None
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
class Facts:
|
||||
"""目录事实的取法:run 时 catalogs 是 context["catalogs"](数据源名 → {表名: notes 或 CatalogEntry}),复算时 recorded 是
|
||||
结论里记下的 params["catalog"]。查过的都记下(record),同一个键只认一次。"""
|
||||
|
||||
def __init__(self, catalogs: Mapping[str, Any] | None = None, recorded: Mapping[str, Any] | None = None) -> None:
|
||||
self._catalogs = catalogs or {}
|
||||
self._recorded = recorded
|
||||
self._labels: dict[str, str | None] = {}
|
||||
self._measures: dict[str, tuple[str | None, bool]] = {}
|
||||
self._dates: dict[str, str | None] = {}
|
||||
|
||||
def _catalog(self, source: str) -> Mapping[str, Any] | None:
|
||||
found = self._catalogs.get(source)
|
||||
return found if isinstance(found, Mapping) else None
|
||||
|
||||
def label(self, snap: Any, column: str) -> str | None:
|
||||
"""结果列给人看的叫法(目录里取自的那一列的中文名);没有为 None。"""
|
||||
if column in self._labels:
|
||||
return self._labels[column]
|
||||
if self._recorded is not None:
|
||||
value = (self._recorded.get("labels") or {}).get(column)
|
||||
else:
|
||||
value = None
|
||||
src = source_of(snap.sql, column)
|
||||
if src is not None and src[1] is not None:
|
||||
item = _item(_table_notes(self._catalog(snap.source), src[0]), src[1], "label")
|
||||
text = item.get("value") if item else None
|
||||
value = text.strip() if isinstance(text, str) and text.strip() else None
|
||||
self._labels[column] = value
|
||||
return value
|
||||
|
||||
def measure(self, snap: Any, column: str) -> tuple[str | None, bool]:
|
||||
"""结果列的度量类型和有没有人确认过:(flow / stock / ratio / identifier / status / attribute 或 None, 确认过)。"""
|
||||
if column in self._measures:
|
||||
return self._measures[column]
|
||||
if self._recorded is not None:
|
||||
kind = (self._recorded.get("measures") or {}).get(column)
|
||||
out = (kind, kind is not None and column not in (self._recorded.get("unconfirmed") or []))
|
||||
else:
|
||||
out = self._live_measure(snap, column)
|
||||
self._measures[column] = out
|
||||
return out
|
||||
|
||||
def _live_measure(self, snap: Any, column: str) -> tuple[str | None, bool]:
|
||||
src = source_of(snap.sql, column)
|
||||
if src is None:
|
||||
return None, False
|
||||
table, name, agg, distinct = src
|
||||
if agg == "count" and not distinct:
|
||||
return "flow", True
|
||||
if agg == "avg":
|
||||
return "ratio", True
|
||||
if agg not in (None, "sum") or name is None:
|
||||
return None, False
|
||||
item = _item(_table_notes(self._catalog(snap.source), table), name, "measure")
|
||||
kind = item.get("value") if item else None
|
||||
if not isinstance(kind, str):
|
||||
return None, False
|
||||
return kind, item.get("status") in _STRONG
|
||||
|
||||
def business_date(self, source: str, table: str) -> str | None:
|
||||
"""表的业务日期列(目录的 business_date);没写为 None。"""
|
||||
key = name_key(table)
|
||||
if key in self._dates:
|
||||
return self._dates[key]
|
||||
if self._recorded is not None:
|
||||
value = (self._recorded.get("dates") or {}).get(key)
|
||||
else:
|
||||
notes = _table_notes(self._catalog(source), table)
|
||||
item = notes.get("business_date") if isinstance(notes, Mapping) else None
|
||||
raw = item.get("value") if isinstance(item, Mapping) else None
|
||||
col = raw.get("column") if isinstance(raw, Mapping) else None
|
||||
value = col if isinstance(col, str) and col.strip() else None
|
||||
self._dates[key] = value
|
||||
return value
|
||||
|
||||
def record(self) -> dict[str, Any] | None:
|
||||
"""查过、查得到的事实(写进 params["catalog"]);一项都没有为 None。"""
|
||||
out: dict[str, Any] = {}
|
||||
labels = {k: v for k, v in self._labels.items() if v is not None}
|
||||
measures = {k: v[0] for k, v in self._measures.items() if v[0] is not None}
|
||||
unconfirmed = sorted(k for k, v in self._measures.items() if v[0] is not None and not v[1])
|
||||
dates = {k: v for k, v in self._dates.items() if v is not None}
|
||||
for key, value in (("labels", labels), ("measures", measures), ("unconfirmed", unconfirmed), ("dates", dates)):
|
||||
if value:
|
||||
out[key] = value
|
||||
return out or None
|
||||
|
||||
|
||||
__all__ = ["Facts", "date_source", "source_of"]
|
||||
@@ -0,0 +1,222 @@
|
||||
"""算子的输入:按名字取快照、拆单元格的写法、按列名认列、遮罩列、度量格换成精确的数、同一键同一期多行(细案 01 §1、§8.1)。
|
||||
|
||||
- 名字:agent 写本节点查过的编号(Q3,和交回模型的「【证据 Q3】」同一个),分析卡写来源节点 id;resolve 取不到就是调用错误。
|
||||
- 单元格写成「名字.r行.列」(Q1.r0.gmv),行从 0 数;名字里可以有点(取最后一个「.r数字.」切开)。
|
||||
- 列名比较不分大小写,按快照的 columns 定位:一字不差的优先;只差大小写的有好几列时是调用错误。
|
||||
- 遮罩列(快照的 masked,不分大小写)不能当维度、度量、期次:调用错误。结论里只出现非遮罩列的格。
|
||||
- 度量格:数(含 DECIMAL 落成的数字文本)换成 Fraction;空值为 None(各算子按细案 §1 处理);文本、布尔是调用错误。
|
||||
查询时记了类型的列按类型认:text 列里的 "2026" 不当数。
|
||||
|
||||
调用错误(contract.CallError)的说法一种,写给 agent:说清哪里不对、怎么改。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
from app.analysis.contract import AnalysisInput, CallError, InputSnapshot
|
||||
from app.analysis.numbers import canon, to_fraction
|
||||
|
||||
#: 单元格的写法:名字.r行.列
|
||||
_CELL = re.compile(r"^(?P<name>.+)\.r(?P<row>\d+)\.(?P<column>.+)$")
|
||||
#: 报错里列出的列最多几个
|
||||
_LIST_MAX = 12
|
||||
|
||||
|
||||
def _listed(columns: Sequence[str]) -> str:
|
||||
shown = "、".join(columns[:_LIST_MAX])
|
||||
return shown + (f"等 {len(columns)} 列" if len(columns) > _LIST_MAX else "")
|
||||
|
||||
|
||||
def hashable(value: Any) -> Any:
|
||||
"""维度值当键:文本、数、布尔、空值原样;别的(列表、对象)和非数、无穷写成文字(同 engine/compare._hashable)。"""
|
||||
if isinstance(value, float) and not math.isfinite(value):
|
||||
return repr(value)
|
||||
return value if isinstance(value, (str, int, float, bool)) or value is None else repr(value)
|
||||
|
||||
|
||||
def same_value(a: Any, b: Any) -> bool:
|
||||
"""调用里写的期次、段和快照里的值是不是同一个:相等,或者写出来一样("2026" 和 2026、" 华东" 和 "华东")。"""
|
||||
if a == b and type(a) is type(b):
|
||||
return True
|
||||
if isinstance(a, bool) or isinstance(b, bool):
|
||||
return a == b
|
||||
fa, fb = to_fraction(a), to_fraction(b)
|
||||
if fa is not None and fb is not None and not isinstance(a, str) and not isinstance(b, str):
|
||||
return fa == fb
|
||||
return str(a).strip() == str(b).strip()
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class Snap:
|
||||
"""一份输入快照(InputSnapshot 的薄包装)。"""
|
||||
|
||||
ref: str
|
||||
data: Mapping[str, Any]
|
||||
dialect: str | None = None
|
||||
columns: list[str] = field(default_factory=list)
|
||||
rows: list[list[Any]] = field(default_factory=list)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.columns = [str(c) for c in self.data.get("columns") or []]
|
||||
self.rows = [list(r) if isinstance(r, (list, tuple)) else [] for r in self.data.get("rows") or []]
|
||||
|
||||
@property
|
||||
def artifact(self) -> str | None:
|
||||
value = self.data.get("artifact")
|
||||
return value if isinstance(value, str) and value else None
|
||||
|
||||
@property
|
||||
def source(self) -> str:
|
||||
return str(self.data.get("source") or "")
|
||||
|
||||
@property
|
||||
def sql(self) -> str:
|
||||
return str(self.data.get("sql") or "")
|
||||
|
||||
@property
|
||||
def truncated(self) -> bool:
|
||||
return bool(self.data.get("truncated"))
|
||||
|
||||
@property
|
||||
def masked(self) -> set[str]:
|
||||
"""不能当维度、度量的列(小写):入口交来的 masked;没交(复算时取回的是快照工件本身)就按快照记下的遮罩列
|
||||
(数据源设的 mask_columns、查询时整列遮掉的 masked_at_query)。"""
|
||||
found = self.data.get("masked")
|
||||
if found is None:
|
||||
found = [*(self.data.get("mask_columns") or []), *(self.data.get("masked_at_query") or [])]
|
||||
return {str(m).strip().lower() for m in found or [] if str(m).strip()}
|
||||
|
||||
def kind(self, column: str) -> str | None:
|
||||
"""查询时记下的这一列的类型(column_types 是 {列: 类型} 或按列序的列表)。"""
|
||||
types = self.data.get("column_types")
|
||||
if isinstance(types, Mapping):
|
||||
kind = types.get(column)
|
||||
elif isinstance(types, list) and column in self.columns:
|
||||
i = self.columns.index(column)
|
||||
kind = types[i] if i < len(types) else None
|
||||
else:
|
||||
kind = None
|
||||
return kind if isinstance(kind, str) else None
|
||||
|
||||
def column(self, wanted: Any, role: str) -> str:
|
||||
"""按名字认一列(不分大小写),认出的是快照里的写法。不存在、只差大小写的好几列、遮罩列都是调用错误。"""
|
||||
if not isinstance(wanted, str) or not wanted.strip():
|
||||
raise CallError("missing_column", f"没写{role}:写 {self.ref} 里的列名")
|
||||
name = wanted.strip()
|
||||
hits = [c for c in self.columns if c == name] or [c for c in self.columns if c.lower() == name.lower()]
|
||||
if not hits:
|
||||
raise CallError("unknown_column", f"{self.ref} 里没有「{name}」这一列,{role}要写它的列名:{self.ref} 的列有 "
|
||||
f"{_listed(self.columns)}")
|
||||
if len(hits) > 1:
|
||||
raise CallError("ambiguous_column", f"{self.ref} 里有好几列都叫「{name}」(只差大小写):{role}写出确切的列名")
|
||||
column = hits[0]
|
||||
if column.lower() in self.masked:
|
||||
raise CallError("masked_column", f"「{column}」是遮罩列,不能当{role}:换一列,或者在 SQL 里不选它")
|
||||
return column
|
||||
|
||||
def index(self, column: str) -> int:
|
||||
return self.columns.index(column)
|
||||
|
||||
def cell(self, row: int, column: str) -> Any:
|
||||
r = self.rows[row]
|
||||
i = self.index(column)
|
||||
return r[i] if i < len(r) else None
|
||||
|
||||
def number(self, row: int, column: str, role: str) -> Fraction | None:
|
||||
"""一格换成精确的数;空值为 None。不是数是调用错误。"""
|
||||
raw = self.cell(row, column)
|
||||
if raw is None:
|
||||
return None
|
||||
if isinstance(raw, bool) or (isinstance(raw, str) and self.kind(column) == "text"):
|
||||
raise self._not_numeric(row, column, raw, role)
|
||||
if isinstance(raw, str):
|
||||
from app.engine.expressions import numeric_text
|
||||
|
||||
if numeric_text(raw.strip()) is None:
|
||||
raise self._not_numeric(row, column, raw, role)
|
||||
value = to_fraction(raw)
|
||||
if value is None:
|
||||
raise self._not_numeric(row, column, raw, role)
|
||||
return value
|
||||
|
||||
def _not_numeric(self, row: int, column: str, raw: Any, role: str) -> CallError:
|
||||
shown = str(raw).replace("\n", " ")[:20]
|
||||
return CallError("not_numeric", f"{self.ref} 的「{column}」不是数值列(第 {row} 行是「{shown}」):{role}要选数值列")
|
||||
|
||||
|
||||
class Inputs:
|
||||
"""按名字取输入,记下结论的 inputs(同一份快照、同一格只记一次,按用到的先后)。"""
|
||||
|
||||
def __init__(self, resolve: Callable[[str], InputSnapshot | None],
|
||||
dialect: Callable[[str, Mapping[str, Any]], str | None] | None = None) -> None:
|
||||
self._resolve = resolve
|
||||
self._dialect = dialect
|
||||
self._snaps: dict[str, Snap] = {}
|
||||
self.records: list[AnalysisInput] = []
|
||||
|
||||
def snap(self, name: Any, role: str) -> Snap:
|
||||
"""按名字取一份快照。"""
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise CallError("missing_query", f"没写{role}:写本节点查过的查询编号(如 Q3)")
|
||||
key = name.strip()
|
||||
if key not in self._snaps:
|
||||
found = self._resolve(key)
|
||||
if not isinstance(found, Mapping):
|
||||
raise CallError("unknown_query", f"没有叫「{key}」的查询结果:{role}写本节点查过的查询编号(如 Q3)")
|
||||
self._snaps[key] = Snap(key, found, self._dialect(key, found) if self._dialect else found.get("dialect"))
|
||||
return self._snaps[key]
|
||||
|
||||
def cell(self, ref: Any, role: str, label: str) -> tuple[Snap, int, str, Fraction | None]:
|
||||
"""「名字.r行.列」→ (快照, 行, 列, 值)。"""
|
||||
found = _CELL.match(ref.strip()) if isinstance(ref, str) else None
|
||||
if found is None:
|
||||
raise CallError("bad_cell", f"{label}要写成单元格「编号.r行.列」(如 Q1.r0.gmv),现在是「{ref}」")
|
||||
snap = self.snap(found.group("name"), label)
|
||||
row = int(found.group("row"))
|
||||
if row >= len(snap.rows):
|
||||
raise CallError("row_out_of_range", f"{snap.ref} 没有第 {row} 行(共 {len(snap.rows)} 行,从 r0 数起):改{label}")
|
||||
column = snap.column(found.group("column"), label)
|
||||
value = snap.number(row, column, label)
|
||||
self.record({"ref": f"{snap.ref}.r{row}.{column}", "role": role, "row": row, "column": column,
|
||||
"value": canon(value)}, snap)
|
||||
return snap, row, column, value
|
||||
|
||||
def use(self, snap: Snap, role: str, columns: Sequence[str], rows: Sequence[int] | None = None) -> None:
|
||||
"""记下一份快照的用法:用了哪几列、哪几行(全用了写 all)。"""
|
||||
used = sorted(set(rows)) if rows is not None else None
|
||||
shown: Any = "all" if used is None or used == list(range(len(snap.rows))) else used
|
||||
self.record({"ref": snap.ref, "role": role, "columns": list(dict.fromkeys(columns)), "rows": shown}, snap)
|
||||
|
||||
def record(self, entry: dict[str, Any], snap: Snap) -> None:
|
||||
out: dict[str, Any] = {"ref": entry["ref"]}
|
||||
if snap.artifact:
|
||||
out["artifact"] = snap.artifact
|
||||
if snap.dialect:
|
||||
out["dialect"] = snap.dialect
|
||||
if snap.masked:
|
||||
out["masked"] = sorted(snap.masked) # 只记列名(复算按它认遮罩列,快照工件里不一定都有)
|
||||
out.update({k: v for k, v in entry.items() if k != "ref"})
|
||||
if out not in self.records:
|
||||
self.records.append(out) # type: ignore[arg-type]
|
||||
|
||||
def parent(self, name: str, artifact: str) -> None:
|
||||
entry: dict[str, Any] = {"ref": name, "artifact": artifact, "role": "parent"}
|
||||
if entry not in self.records:
|
||||
self.records.append(entry) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def as_list(value: Any) -> list[Any]:
|
||||
"""参数里一个值或一组值:都规范成列表(None 是空列表)。"""
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, (list, tuple)):
|
||||
return list(value)
|
||||
return [value]
|
||||
|
||||
|
||||
__all__ = ["Inputs", "Snap", "as_list", "hashable", "same_value"]
|
||||
@@ -0,0 +1,143 @@
|
||||
"""分析结论里的数:精确的有理数进、规范的十进制字符串出(细案 01 §1「数用十进制」)。
|
||||
|
||||
结论要能逐字节复算(证据面板、导出、可验证报告的复算和封存的逐字段比,评测作答前后各算一次逐字比),所以:
|
||||
|
||||
- **算**:快照里的数一律换成 Fraction(浮点按最短写法 repr,数字文本按字面,DECIMAL 落成的文本不经浮点),加减乘除全程
|
||||
精确——各段之和逐位等于总变化,Σ结构效应 + Σ自身效应逐位等于比率的变化。只有要开方、取对数、查分布的统计量是浮点。
|
||||
- **写**:canon 把数写成规范的十进制字符串:不带指数、不带正号、去掉多余的 0、没有「-0」。能精确写成有限小数的(快照里的
|
||||
数和它们的和差,0.25 这样的商)照写;除不尽的和浮点按 SIG 位有效数字四舍五入(ROUND_HALF_UP)。浮点只留 12 位有效
|
||||
数字:不同机器的数学库在最后一两位上可能不一样,留 12 位同一份输入在哪台机器上复算都是同一串字。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
#: 除不尽的数、浮点保留几位有效数字
|
||||
SIG = 12
|
||||
#: 能精确写成有限小数、但有效数字超过这么多位的,也按 SIG 位写(只会出现在反复除以 2 这类极端情形)
|
||||
_EXACT_MAX_DIGITS = 30
|
||||
|
||||
|
||||
def to_fraction(value: Any) -> Fraction | None:
|
||||
"""一个数换成精确的 Fraction:整数、Decimal、Fraction 原样;浮点按最短写法(repr);数字文本按字面。布尔、非数、无穷、
|
||||
认不出的为 None。"""
|
||||
if isinstance(value, bool) or value is None:
|
||||
return None
|
||||
if isinstance(value, Fraction):
|
||||
return value
|
||||
if isinstance(value, int):
|
||||
return Fraction(value)
|
||||
if isinstance(value, float):
|
||||
if not math.isfinite(value):
|
||||
return None
|
||||
return Fraction(Decimal(repr(value)))
|
||||
if isinstance(value, Decimal):
|
||||
return Fraction(value) if value.is_finite() else None
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
d = Decimal(value.strip())
|
||||
except (InvalidOperation, ValueError):
|
||||
return None
|
||||
return Fraction(d) if d.is_finite() else None
|
||||
return None
|
||||
|
||||
|
||||
def _terminating(q: Fraction) -> int | None:
|
||||
"""q 能精确写成有限小数时要几位小数;不能为 None。"""
|
||||
d = q.denominator
|
||||
twos = fives = 0
|
||||
while d % 2 == 0:
|
||||
d //= 2
|
||||
twos += 1
|
||||
while d % 5 == 0:
|
||||
d //= 5
|
||||
fives += 1
|
||||
return max(twos, fives) if d == 1 else None
|
||||
|
||||
|
||||
def _plain(digits: int, exp10: int) -> str:
|
||||
"""digits × 10^exp10(digits ≥ 0)写成不带指数、去尾零的十进制。"""
|
||||
text = str(digits)
|
||||
if exp10 >= 0:
|
||||
return text + "0" * exp10 if digits else "0"
|
||||
places = -exp10
|
||||
text = text.rjust(places + 1, "0")
|
||||
head, tail = text[:-places], text[-places:].rstrip("0")
|
||||
return f"{head}.{tail}" if tail else head
|
||||
|
||||
|
||||
def _decade(a: Fraction) -> int:
|
||||
"""a > 0 的十进制量级:10^e ≤ a < 10^(e+1) 的 e。"""
|
||||
try:
|
||||
e = math.floor(math.log10(a.numerator) - math.log10(a.denominator))
|
||||
except (ValueError, OverflowError):
|
||||
e = len(str(a.numerator)) - len(str(a.denominator))
|
||||
while Fraction(10) ** e > a:
|
||||
e -= 1
|
||||
while Fraction(10) ** (e + 1) <= a:
|
||||
e += 1
|
||||
return e
|
||||
|
||||
|
||||
def round_sig(q: Fraction, sig: int = SIG) -> Fraction:
|
||||
"""按 sig 位有效数字四舍五入(ROUND_HALF_UP)。"""
|
||||
if q == 0:
|
||||
return Fraction(0)
|
||||
sign = -1 if q < 0 else 1
|
||||
a = abs(q)
|
||||
e = _decade(a)
|
||||
scale = Fraction(10) ** (sig - 1 - e)
|
||||
m = math.floor(a * scale + Fraction(1, 2))
|
||||
return sign * Fraction(m) / scale
|
||||
|
||||
|
||||
def round_places(q: Fraction, places: int) -> Fraction:
|
||||
"""按 places 位小数四舍五入(ROUND_HALF_UP,负数按绝对值舍入,同 Decimal 的 ROUND_HALF_UP)。"""
|
||||
sign = -1 if q < 0 else 1
|
||||
scale = Fraction(10) ** places
|
||||
return sign * Fraction(math.floor(abs(q) * scale + Fraction(1, 2))) / scale
|
||||
|
||||
|
||||
def _write(q: Fraction) -> str:
|
||||
"""有限小数的 q 写出来(调用方保证 q 能精确写成有限小数)。"""
|
||||
places = _terminating(q) or 0
|
||||
sign = "-" if q < 0 else ""
|
||||
digits = abs(q) * (10 ** places)
|
||||
assert digits.denominator == 1
|
||||
return sign + _plain(int(digits), -places)
|
||||
|
||||
|
||||
def canon(value: Any) -> str | None:
|
||||
"""规范的十进制字符串(见文件头)。None、非数、无穷为 None。"""
|
||||
if isinstance(value, float) and math.isfinite(value):
|
||||
q = round_sig(to_fraction(value) or Fraction(0))
|
||||
else:
|
||||
q = to_fraction(value)
|
||||
if q is None:
|
||||
return None
|
||||
places = _terminating(q)
|
||||
if places is None or len(str(int(abs(q) * 10 ** places)).rstrip("0")) > _EXACT_MAX_DIGITS:
|
||||
q = round_sig(q)
|
||||
if q == 0:
|
||||
return "0"
|
||||
return _write(q)
|
||||
|
||||
|
||||
def fixed(q: Fraction, places: int, *, strip: bool = True, group: bool = False) -> str:
|
||||
"""按 places 位小数四舍五入写出来;strip 去掉尾零,group 加千分位。没有正号;舍成 0 的写「0」(不写「-0」)。"""
|
||||
r = round_places(q, places)
|
||||
sign = "-" if r < 0 else ""
|
||||
scaled = abs(r) * (10 ** places)
|
||||
whole, frac = divmod(int(scaled), 10 ** places)
|
||||
head = f"{whole:,}" if group else str(whole)
|
||||
tail = str(frac).rjust(places, "0") if places > 0 else ""
|
||||
if strip:
|
||||
tail = tail.rstrip("0")
|
||||
text = f"{head}.{tail}" if tail else head
|
||||
return sign + text
|
||||
|
||||
|
||||
__all__ = ["SIG", "canon", "fixed", "round_places", "round_sig", "to_fraction"]
|
||||
@@ -0,0 +1,224 @@
|
||||
"""期次:快照里期次列的值认成可以排序、能判断连不连续、知道覆盖哪几天的期(细案 01 §3「期次认得出、能排序」)。
|
||||
|
||||
认这几种写法(一列里要是同一种):
|
||||
|
||||
- 日期:DATE / DATETIME(零点)、'YYYY-MM-DD'、'YYYYMMDD'、8 位整数。粒度不写时按值推断:都是某月 1 日(或都是月末)
|
||||
且相邻两期隔整月 → month(月份都在 1、4、7、10 月且隔整季 → quarter);相邻两期都隔 7 天的整数倍 → week(这个日期当
|
||||
一周的第一天);否则 day。
|
||||
- 月份:'YYYY-MM'、'YYYYMM'(6 位,月份 01–12)→ month。
|
||||
- ISO 周:'YYYY-Www'(2026-W42 是 2026-10-12 到 10-18)→ week。SQLite 的 strftime('%W') 不是 ISO 周号(年初那几天
|
||||
算第 0 周),按它写的期次覆盖的日子会差一周:方法里要求期次写成每周第一天的日期。
|
||||
- 季度:'YYYY-Qn'、'YYYYQn' → quarter。
|
||||
- 整数序号(不是上面几种的整数)→ seq:能排序、能判连续,不知道覆盖哪几天(本期完不完整核对不了)。
|
||||
|
||||
带时刻(非零点)的、写法混着的、认不出的都算认不出(period_unreadable)。
|
||||
|
||||
每一期记 ordinal:同一粒度里相邻两期的 ordinal 差 1(日:天数,周:从第一期起第几个 7 天,月:年 × 12 + 月,季:年 × 4 +
|
||||
季,序号:本身),据此排序、判缺期、按 lag 找比较期。start / end 是这一期覆盖的第一天和最后一天(含),seq 没有。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import calendar
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from app.data import coverage
|
||||
|
||||
GRAINS = ("day", "week", "month", "quarter")
|
||||
#: 粒度的叫法:数量词(过去 12 周、12 个月)、每…(每周、每月)
|
||||
COUNT_WORD = {"day": "天", "week": "周", "month": "个月", "quarter": "个季度", "seq": "期"}
|
||||
PER_WORD = {"day": "天", "week": "周", "month": "月", "quarter": "季度", "seq": "期"}
|
||||
#: 粒度不写时各粒度的缺省 lag:日数据和上周同一天比(避开星期效应),其余和上一期比
|
||||
DEFAULT_LAG = {"day": 7, "week": 1, "month": 1, "quarter": 1, "seq": 1}
|
||||
|
||||
_MONTH = re.compile(r"^(\d{4})-?(\d{2})$")
|
||||
_WEEK = re.compile(r"^(\d{4})-?W(\d{1,2})$", re.I)
|
||||
_QUARTER = re.compile(r"^(\d{4})-?Q([1-4])$", re.I)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Period:
|
||||
raw: Any # 快照里的原值
|
||||
ordinal: int
|
||||
start: date | None
|
||||
end: date | None
|
||||
|
||||
|
||||
def _month_end(d: date) -> date:
|
||||
return d.replace(day=calendar.monthrange(d.year, d.month)[1])
|
||||
|
||||
|
||||
def _quarter_start(d: date) -> date:
|
||||
return date(d.year, 3 * ((d.month - 1) // 3) + 1, 1)
|
||||
|
||||
|
||||
def _quarter_end(d: date) -> date:
|
||||
start = _quarter_start(d)
|
||||
return _month_end(date(start.year, start.month + 2, 1))
|
||||
|
||||
|
||||
def _as_date(value: Any) -> date | None:
|
||||
"""日期写法的期次 → 那一天;带非零点时刻的、认不出的为 None。"""
|
||||
if isinstance(value, datetime):
|
||||
return value.date() if value.time() == datetime.min.time() else None
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
day = coverage.as_day(value)
|
||||
if day is None or day.timed:
|
||||
return None
|
||||
return day.day
|
||||
|
||||
|
||||
def _shape(value: Any) -> str | None:
|
||||
"""一个期次值是哪种写法:date / month / week / quarter / seq;认不出为 None。"""
|
||||
if isinstance(value, bool) or value is None:
|
||||
return None
|
||||
if isinstance(value, (date, datetime)):
|
||||
return "date" if _as_date(value) else None
|
||||
if isinstance(value, float) and value.is_integer():
|
||||
value = int(value)
|
||||
if isinstance(value, int):
|
||||
text = str(value)
|
||||
if len(text) == 8 and _as_date(value):
|
||||
return "date"
|
||||
if len(text) == 6 and (m := _MONTH.match(text)) and 1 <= int(m.group(2)) <= 12:
|
||||
return "month"
|
||||
return "seq"
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
text = value.strip()
|
||||
if (m := _MONTH.match(text)) and 1 <= int(m.group(2)) <= 12 and 1900 <= int(m.group(1)) <= 2200:
|
||||
return "month"
|
||||
if _WEEK.match(text):
|
||||
return "week"
|
||||
if _QUARTER.match(text):
|
||||
return "quarter"
|
||||
if _as_date(text):
|
||||
return "date"
|
||||
if re.fullmatch(r"-?\d+", text):
|
||||
return "seq"
|
||||
return None
|
||||
|
||||
|
||||
def _month_of(value: Any) -> date:
|
||||
m = _MONTH.match(str(int(value) if isinstance(value, (int, float)) else value).strip())
|
||||
assert m is not None
|
||||
return date(int(m.group(1)), int(m.group(2)), 1)
|
||||
|
||||
|
||||
def _infer_date_grain(days: Sequence[date]) -> str:
|
||||
"""一列日期的粒度(见文件头)。"""
|
||||
ordered = sorted(set(days))
|
||||
if len(ordered) < 2:
|
||||
return "day"
|
||||
firsts = all(d.day == 1 for d in ordered)
|
||||
lasts = all(d == _month_end(d) for d in ordered)
|
||||
if firsts or lasts:
|
||||
months = [d.year * 12 + d.month - 1 for d in ordered]
|
||||
gaps = [b - a for a, b in zip(months, months[1:])]
|
||||
if all(g >= 1 for g in gaps):
|
||||
if all(d.month in (1, 4, 7, 10) for d in ordered) and all(g % 3 == 0 for g in gaps) and firsts:
|
||||
return "quarter"
|
||||
return "month"
|
||||
gaps = [(b - a).days for a, b in zip(ordered, ordered[1:])]
|
||||
if all(g % 7 == 0 for g in gaps):
|
||||
return "week"
|
||||
return "day"
|
||||
|
||||
|
||||
def parse(values: Sequence[Any], grain: str | None = None) -> tuple[list[Period], str] | None:
|
||||
"""一列期次值 → (逐个的 Period,粒度)。认不出(写法不一、带时刻、粒度和写法对不上)返回 None。"""
|
||||
shapes = {_shape(v) for v in values}
|
||||
if not values or None in shapes or len(shapes) != 1:
|
||||
return None
|
||||
shape = shapes.pop()
|
||||
if shape == "seq":
|
||||
if grain not in (None, "seq"):
|
||||
return None
|
||||
return [Period(v, int(v), None, None) for v in values], "seq"
|
||||
if shape == "month":
|
||||
if grain not in (None, "month"):
|
||||
return None
|
||||
out = []
|
||||
for v in values:
|
||||
start = _month_of(v)
|
||||
out.append(Period(v, start.year * 12 + start.month - 1, start, _month_end(start)))
|
||||
return out, "month"
|
||||
if shape == "week":
|
||||
if grain not in (None, "week"):
|
||||
return None
|
||||
out = []
|
||||
for v in values:
|
||||
m = _WEEK.match(str(v).strip())
|
||||
assert m is not None
|
||||
try:
|
||||
start = date.fromisocalendar(int(m.group(1)), int(m.group(2)), 1)
|
||||
except ValueError:
|
||||
return None
|
||||
out.append(Period(v, start.toordinal() // 7, start, start + timedelta(days=6)))
|
||||
return out, "week"
|
||||
if shape == "quarter":
|
||||
if grain not in (None, "quarter"):
|
||||
return None
|
||||
out = []
|
||||
for v in values:
|
||||
m = _QUARTER.match(str(v).strip())
|
||||
assert m is not None
|
||||
start = date(int(m.group(1)), 3 * (int(m.group(2)) - 1) + 1, 1)
|
||||
out.append(Period(v, start.year * 4 + (start.month - 1) // 3, start, _quarter_end(start)))
|
||||
return out, "quarter"
|
||||
days = [_as_date(v) for v in values]
|
||||
assert all(days)
|
||||
grain = grain or _infer_date_grain(days) # type: ignore[arg-type]
|
||||
if grain == "day":
|
||||
return [Period(v, d.toordinal(), d, d) for v, d in zip(values, days)], "day" # type: ignore[union-attr]
|
||||
if grain == "week":
|
||||
first = min(days) # type: ignore[type-var]
|
||||
if any((d - first).days % 7 for d in days): # type: ignore[operator]
|
||||
return None
|
||||
return [Period(v, (d - first).days // 7, d, d + timedelta(days=6)) # type: ignore[operator]
|
||||
for v, d in zip(values, days)], "week"
|
||||
if grain == "month":
|
||||
if not all(d.day == 1 or d == _month_end(d) for d in days): # type: ignore[union-attr]
|
||||
return None
|
||||
return [Period(v, d.year * 12 + d.month - 1, d.replace(day=1), _month_end(d)) # type: ignore[union-attr]
|
||||
for v, d in zip(values, days)], "month"
|
||||
if grain == "quarter":
|
||||
if not all(d.day == 1 and d.month in (1, 4, 7, 10) for d in days): # type: ignore[union-attr]
|
||||
return None
|
||||
return [Period(v, d.year * 4 + (d.month - 1) // 3, d, _quarter_end(d)) # type: ignore[union-attr]
|
||||
for v, d in zip(values, days)], "quarter"
|
||||
return None
|
||||
|
||||
|
||||
def calendar_unit(start: date, end: date) -> str | None:
|
||||
"""这段日子是不是一个整的日历期:整月 month、整季 quarter、整年 year;都不是为 None。整月对整月算一样长(31 天的 10 月
|
||||
和 30 天的 9 月环比是通行的比法),不按天数判窗口不等。"""
|
||||
if start.day != 1:
|
||||
return None
|
||||
if start.month == 1 and end == date(start.year, 12, 31):
|
||||
return "year"
|
||||
if start.month in (1, 4, 7, 10) and end == _quarter_end(start):
|
||||
return "quarter"
|
||||
if end == _month_end(start):
|
||||
return "month"
|
||||
return None
|
||||
|
||||
|
||||
def weekend_days(start: date, end: date) -> int:
|
||||
"""[start, end] 里周六、周日的天数。"""
|
||||
days = (end - start).days + 1
|
||||
full, rest = divmod(days, 7)
|
||||
count = 2 * full
|
||||
for k in range(rest):
|
||||
if (start + timedelta(days=7 * full + k)).weekday() >= 5:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
__all__ = ["COUNT_WORD", "DEFAULT_LAG", "GRAINS", "PER_WORD", "Period", "calendar_unit", "parse", "weekend_days"]
|
||||
@@ -0,0 +1,517 @@
|
||||
"""判定语、判定标准、附带说明:一种说法,只在这里定义(细案 01 §7)。
|
||||
|
||||
判定语由系统渲染([[a:A3]] 渲染成结论里的 phrase),写作者只引用、不改写——「叙述层无算术」往前一步:分析判定和数字一样
|
||||
由系统写。前后端都只读结论里渲染好的字符串,所以模板、数的写法、附带说明的那一句都在这里,算子只填关节。
|
||||
|
||||
**数的写法**(和 engine/evidence.render_cell 同一种风格:Decimal 式的四舍五入 ROUND_HALF_UP,千分位用「,」):
|
||||
|
||||
- 百分比:一位小数、带正负号(+8.7%、-3.1%,0 写 0.0%);占比、比重不带正号(占 61.9%)。不是 0 的值写出来成了 0.0%
|
||||
的,多给小数位(最多 4 位),不写一个假的 0。
|
||||
- 量(金额、件数):千分位,最多两位小数、去尾零;绝对值小于 1 的保留两位有效数字(最多 6 位小数),同样不写假的 0。
|
||||
- 百分点:比率的变化(2.1% 到 2.5% 是上升 0.4 个百分点)。
|
||||
|
||||
**label 消毒**(§7.3):判定语里的度量叫法、因子名、维度叫法是 agent 或数据给的。agent 给的 label(以及目录里的叫法、
|
||||
当叫法用的列名)去掉数字、百分号、单位符号、引用标记,不超过 20 字,消毒后为空就退回下一个来源:不然模型可以把
|
||||
「销售额(同比 +30%)」塞进 label,借系统渲染绕过裸数字检测。去数字按出具校验抽数字的同一个函数(engine/issuance 的
|
||||
extract_numbers:阿拉伯数字、中文数字、带万 / 亿 / 成 / 倍的写法),再去掉剩下的数字字符。维度值是数据,照原样。
|
||||
|
||||
**附带说明**:判定语后面用「;」接,最多两条(按 PHRASE_ORDER 的先后),其余只在证据面板。度量类型没标、以各段之和为
|
||||
总数这类方法上的交代不进判定语。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
from app.analysis.numbers import fixed, to_fraction
|
||||
|
||||
#: 算子的叫法:「算不了」的判定语、报错都用它
|
||||
OP_NAMES = {"decompose": "变化拆解", "baseline": "波动基线", "trend": "趋势判断", "differ": "差异检验",
|
||||
"comparable": "可比性检查"}
|
||||
#: 附带说明进判定语的先后;不在这里的(measure_unlabeled、headline_basis)只在证据面板
|
||||
PHRASE_ORDER = ("comparability", "simpson", "offset", "outlier_sensitive", "small_sample", "record_high", "record_low",
|
||||
"multiple_comparisons", "weekday_effect", "small_denominator", "null_values")
|
||||
#: 判定语里最多接几条附带说明
|
||||
PHRASE_CAVEATS = 2
|
||||
#: 消毒后的叫法最多几个字
|
||||
LABEL_MAX = 20
|
||||
#: 维度键有两列时,一段的写法(华东·线上)
|
||||
KEY_JOIN = "·"
|
||||
#: 维度值为空时的写法
|
||||
EMPTY_KEY = "(空)"
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 拼字:汉字和英文、数字之间加一个空格,挨着全角标点的不加(同 data/sql_semantics._say)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_CJK_PUNCT = frozenset(",。;:、!?「」『』()《》【】…—·")
|
||||
|
||||
|
||||
def _wide(ch: str) -> bool:
|
||||
return "⺀" <= ch <= "鿿" or "豈" <= ch <= "" or "" <= ch <= ""
|
||||
|
||||
|
||||
def say(*parts: str | None) -> str:
|
||||
"""拼一句话:汉字和英文、数字之间补一个空格,挨着全角标点、空白的不补,汉字和汉字之间不补。"""
|
||||
out = ""
|
||||
for part in parts:
|
||||
if not part:
|
||||
continue
|
||||
if out and not out[-1].isspace() and not part[0].isspace():
|
||||
a, b = out[-1], part[0]
|
||||
if not (a in _CJK_PUNCT or b in _CJK_PUNCT or (_wide(a) and _wide(b)) or (not _wide(a) and not _wide(b))):
|
||||
out += " "
|
||||
out += part
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 数
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _q(value: Any) -> Fraction:
|
||||
q = value if isinstance(value, Fraction) else to_fraction(value)
|
||||
if q is None:
|
||||
raise ValueError(f"不是数:{value!r}")
|
||||
return q
|
||||
|
||||
|
||||
def _nonzero_fixed(q: Fraction, places: int, *, most: int, strip: bool, group: bool = False) -> str:
|
||||
"""按 places 位小数写;不是 0 的值写出来成了 0 的,逐位多给,最多 most 位。"""
|
||||
for p in range(places, most + 1):
|
||||
text = fixed(q, p, strip=strip if p == places else True, group=group)
|
||||
if q == 0 or text.strip("-0.,"):
|
||||
return text
|
||||
return fixed(q, most, strip=True, group=group)
|
||||
|
||||
|
||||
def pct(value: Any, *, signed: bool = True) -> str:
|
||||
"""比率写成百分比:一位小数;signed 时正数带「+」(0 不带)。"""
|
||||
q = _q(value) * 100
|
||||
text = _nonzero_fixed(q, 1, most=4, strip=False)
|
||||
if signed and q > 0 and not text.startswith("-"):
|
||||
text = "+" + text
|
||||
return text + "%"
|
||||
|
||||
|
||||
def points(value: Any) -> str:
|
||||
"""比率的变化写成百分点(不带符号,调用方写上升 / 下降):0.004 → 0.4。"""
|
||||
return _nonzero_fixed(abs(_q(value)) * 100, 1, most=4, strip=True)
|
||||
|
||||
|
||||
def amount(value: Any, *, signed: bool = False) -> str:
|
||||
"""量:千分位,最多两位小数、去尾零;绝对值小于 1 的保留两位有效数字(最多 6 位小数)。signed 时正数带「+」。"""
|
||||
q = _q(value)
|
||||
places = 2
|
||||
if q != 0 and abs(q) < 1:
|
||||
places = max(2, -math.floor(math.log10(float(abs(q)))) + 1)
|
||||
places = min(places, 6)
|
||||
text = _nonzero_fixed(q, places, most=6, strip=True, group=True)
|
||||
if signed and q > 0:
|
||||
text = "+" + text
|
||||
return text
|
||||
|
||||
|
||||
def threshold_pct(value: Any) -> str:
|
||||
"""阈值写成百分比:去尾零、不带正号(0.005 → 0.5%,0.8 → 80%)。"""
|
||||
return fixed(_q(value) * 100, 4, strip=True) + "%"
|
||||
|
||||
|
||||
def share_words(value: Any) -> str:
|
||||
"""阈值写进判定标准:0.5 写「一半」,其余写百分比。"""
|
||||
q = _q(value)
|
||||
return "一半" if q == Fraction(1, 2) else threshold_pct(q)
|
||||
|
||||
|
||||
def number(value: Any) -> str:
|
||||
"""阈值本身(倍数、显著性水平、期数):去尾零、不带千分位。"""
|
||||
return fixed(_q(value), 6, strip=True)
|
||||
|
||||
|
||||
def change(delta: Any, base: Any) -> str:
|
||||
"""一个量的变化:下降 194(-8.7%)/ 上升 120(+5.3%);基期为 0 不给百分比;没变写「没有变化」。"""
|
||||
d = _q(delta)
|
||||
if d == 0:
|
||||
return "没有变化"
|
||||
word = "上升" if d > 0 else "下降"
|
||||
b = _q(base) if base is not None else Fraction(0)
|
||||
tail = f"({pct(d / abs(b))})" if b != 0 else ""
|
||||
return f"{word} {amount(abs(d))}{tail}"
|
||||
|
||||
|
||||
def ratio_change(r_base: Any, r_target: Any, *, proportion: bool) -> str:
|
||||
"""一个比率的变化。proportion(各段的分子都不超过分母,比率是一个占比,如退款率)写百分点:上升 0.4 个百分点(2.1% 到
|
||||
2.5%);其余(客单价这类)按量写:上升 3.2(+3.9%)。"""
|
||||
rb, rt = _q(r_base), _q(r_target)
|
||||
d = rt - rb
|
||||
if not proportion:
|
||||
return change(d, rb)
|
||||
if d == 0:
|
||||
return f"没有变化({pct(rt, signed=False)})"
|
||||
word = "上升" if d > 0 else "下降"
|
||||
return f"{word} {points(d)} 个百分点({pct(rb, signed=False)} 到 {pct(rt, signed=False)})"
|
||||
|
||||
|
||||
def ratio_value(value: Any, *, proportion: bool) -> str:
|
||||
"""比率本身:占比写百分比,其余按量写。"""
|
||||
return pct(value, signed=False) if proportion else amount(value)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 叫法与段
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_MARKER = re.compile(r"\[\[[^\]\n]*\]\]|\[\[|\]\]")
|
||||
_DIGITS = re.compile(r"[0-90-9]")
|
||||
_SYMBOLS = re.compile(r"[%%‰‱¥¥$$€£₩℃°++\-−–±×÷==<><>~~^*]")
|
||||
_EMPTY_BRACKETS = re.compile(r"[((【\[]\s*[))】\]]")
|
||||
_EDGE_PUNCT = ",,、::;;。.!!?? "
|
||||
|
||||
|
||||
def clean_label(text: Any) -> str:
|
||||
"""消毒(见文件头):去引用标记、数字(出具校验认的数字写法连同后缀,再加剩下的数字字符)、百分号和单位符号、空括号,
|
||||
压掉多余空白,不超过 LABEL_MAX 字。可能消成空串。"""
|
||||
if not isinstance(text, str):
|
||||
return ""
|
||||
from app.engine.issuance import extract_numbers
|
||||
|
||||
raw = _MARKER.sub("", text)
|
||||
spans = sorted((n.start, n.end) for n in extract_numbers(raw))
|
||||
kept, pos = [], 0
|
||||
for start, end in spans:
|
||||
if start >= pos:
|
||||
kept.append(raw[pos:start])
|
||||
pos = end
|
||||
kept.append(raw[pos:])
|
||||
out = _SYMBOLS.sub("", _DIGITS.sub("", "".join(kept)))
|
||||
# 去掉数字以后留下的空白、标点:括号、标点两边的空白,括号里头尾的标点,连着的几个标点
|
||||
out = re.sub(r"\s*([()()【】\[\],,、::;;])\s*", r"\1", out)
|
||||
out = re.sub(r"[,,、::;;]+(?=[))】\]])", "", out)
|
||||
out = re.sub(r"(?<=[((【\[])[,,、::;;]+", "", out)
|
||||
out = re.sub(r"([,,、::;;])[,,、::;;]+", r"\1", out)
|
||||
while _EMPTY_BRACKETS.search(out):
|
||||
out = _EMPTY_BRACKETS.sub("", out)
|
||||
out = re.sub(r"\s+", " ", out).strip(_EDGE_PUNCT)
|
||||
return out[:LABEL_MAX].strip(_EDGE_PUNCT)
|
||||
|
||||
|
||||
def pick_label(*candidates: Any, fallback: str = "这一列") -> str:
|
||||
"""依次取第一个消毒后不为空的叫法;都为空用 fallback。"""
|
||||
for c in candidates:
|
||||
cleaned = clean_label(c)
|
||||
if cleaned:
|
||||
return cleaned
|
||||
return fallback
|
||||
|
||||
|
||||
def key_text(key: Sequence[Any] | Any) -> str:
|
||||
"""一段(维度键)的写法:一列就是那个值,两列用「·」连(华东·线上);空值写「(空)」。数据里的值照原样(压成一行)。"""
|
||||
parts = key if isinstance(key, (list, tuple)) else [key]
|
||||
shown = []
|
||||
for p in parts:
|
||||
if p is None:
|
||||
shown.append(EMPTY_KEY)
|
||||
elif isinstance(p, bool):
|
||||
shown.append("是" if p else "否")
|
||||
else:
|
||||
shown.append(re.sub(r"\s+", " ", str(p)).strip() or EMPTY_KEY)
|
||||
return KEY_JOIN.join(shown)
|
||||
|
||||
|
||||
def names(items: Iterable[str], limit: int | None = None) -> str:
|
||||
"""几段的写法:华东、华北;超过 limit 段时写前 limit 段加「等 n 段」。"""
|
||||
items = list(items)
|
||||
if limit is not None and len(items) > limit:
|
||||
return "、".join(items[:limit]) + f"等 {len(items)} 段"
|
||||
return "、".join(items)
|
||||
|
||||
|
||||
def dims_text(labels: Sequence[str]) -> str:
|
||||
"""维度的叫法:地区;两列时写「地区、渠道」。"""
|
||||
return "、".join(labels)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 附带说明(caveats[].text):一种说法
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def caveat(code: str, text: str) -> dict[str, str]:
|
||||
return {"code": code, "text": text}
|
||||
|
||||
|
||||
def offset_text(delta_total: Any, gross_up: Any, gross_down: Any) -> str:
|
||||
"""抵消:和总变化方向相反的段合计多少(下降的段合计 -120,上升的段合计 +40,互相抵消了一部分)。"""
|
||||
up, down = amount(gross_up, signed=True), amount(gross_down, signed=True)
|
||||
if _q(delta_total) >= 0:
|
||||
return f"上升的段合计 {up},下降的段合计 {down},互相抵消了一部分"
|
||||
return f"下降的段合计 {down},上升的段合计 {up},互相抵消了一部分"
|
||||
|
||||
|
||||
def simpson_text(label: str, overall_up: bool, subject: str, w_base: Any, w_target: Any) -> str:
|
||||
"""辛普森:各段的比率大多和整体反着变,整体的变化来自结构。"""
|
||||
seg = "上升" if not overall_up else "下降"
|
||||
whole = "上升" if overall_up else "下降"
|
||||
move = "升" if _q(w_target) > _q(w_base) else "降"
|
||||
return say(f"各段的", label, f"大多在{seg},整体却{whole}:变化来自结构,", subject,
|
||||
f"占比从 {pct(w_base, signed=False)} {move}到 {pct(w_target, signed=False)}")
|
||||
|
||||
|
||||
def small_denominator_text(segments: Sequence[str], denominator: str, minimum: int) -> str:
|
||||
return say(names(segments, 3), "的", denominator, f"不到 {minimum},比率波动大,不参与多数判断")
|
||||
|
||||
|
||||
def null_segments_text(segments: Sequence[str], label: str) -> str:
|
||||
return say(names(segments, 3), "的", label, "为空,按 0 算")
|
||||
|
||||
|
||||
def null_periods_text(count: int, label: str) -> str:
|
||||
return say(f"有 {count} 期的", label, "为空,跳过了这几期")
|
||||
|
||||
|
||||
def unlabeled_text(column: str, tail: str) -> str:
|
||||
return say("目录没标「", column, "」的度量类型,", tail)
|
||||
|
||||
|
||||
def unconfirmed_text(column: str, measure: str, tail: str) -> str:
|
||||
return say("目录里「", column, f"」是{MEASURE_WORDS.get(measure, measure)}只是推断,没人确认,", tail)
|
||||
|
||||
|
||||
MEASURE_WORDS = {"flow": "流量", "stock": "存量", "ratio": "比率", "identifier": "标识", "status": "状态", "attribute": "属性"}
|
||||
HEADLINE_BASIS = "没给总数,以各段之和为总数"
|
||||
WEEKDAY_EFFECT = "日数据有星期效应,看趋势建议按周汇总"
|
||||
|
||||
|
||||
def record_text(high: bool, label: str, n: int, unit: str) -> str:
|
||||
return say("本期的", label, f"是过去 {n} {unit}里{'最高' if high else '最低'}的")
|
||||
|
||||
|
||||
def multiple_text(pairs: int) -> str:
|
||||
return f"比了 {pairs} 对,已按次数校正"
|
||||
|
||||
|
||||
def outlier_text(largest: bool, now_differs: bool) -> str:
|
||||
which = "最大" if largest else "最小"
|
||||
return f"去掉{which}的一笔后,差异{'变得站得住' if now_differs else '不再站得住'}"
|
||||
|
||||
|
||||
def small_sample_rate_text(minimum: int) -> str:
|
||||
return f"样本少(有格子的期望频数不到 {minimum}),用了精确检验"
|
||||
|
||||
|
||||
def small_sample_mean_text(group: str, n: int, minimum: int) -> str:
|
||||
return say("有一组的样本数不到 ", str(minimum), "(", group, f" {n}),结论不稳")
|
||||
|
||||
|
||||
UNVERIFIED_COMPLETE = "没能核对本期的数据是否到齐(查不到表里这一列的日期范围)"
|
||||
|
||||
|
||||
def with_caveats(main: str, caveats: Sequence[Mapping[str, str]], *, skip: Iterable[str] = ()) -> str:
|
||||
"""判定语接上最多 PHRASE_CAVEATS 条附带说明(按 PHRASE_ORDER 的先后,用「;」接)。skip 里的那几条(已经在正文里说过的、
|
||||
只在证据面板说的)不接。"""
|
||||
skipped = set(skip)
|
||||
ranked = sorted((PHRASE_ORDER.index(c["code"]), i, c["text"]) for i, c in enumerate(caveats)
|
||||
if c["code"] in PHRASE_ORDER and c["code"] not in skipped)
|
||||
texts = list(dict.fromkeys(t for _, _, t in ranked))[:PHRASE_CAVEATS]
|
||||
return ";".join([main, *texts])
|
||||
|
||||
|
||||
def refused(op: str, reason: str) -> str:
|
||||
"""算不了的判定语:{算子的叫法}没有做成:{原因}。"""
|
||||
return f"{OP_NAMES[op]}没有做成:{reason}"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 判定语模板(§7.2)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def decompose_driver(label: str, total: str, subject: str, subject_change: str, share: Any) -> str:
|
||||
return say(label, total, ",主要来自", subject, f"({subject_change},占 {pct(share, signed=False)})")
|
||||
|
||||
|
||||
def decompose_concentrated(label: str, total: str, subjects: str, share: Any) -> str:
|
||||
return say(label, total, ",集中在", subjects, f"(合计占 {pct(share, signed=False)})")
|
||||
|
||||
|
||||
def decompose_dispersed(label: str, total: str, dims: str, main_share: Any) -> str:
|
||||
return say(label, total, ",分散在各", dims, ",没有哪一段占到", share_words(main_share))
|
||||
|
||||
|
||||
def decompose_flat(label: str, pct_total: Any | None, offset: tuple[str, str, str, str] | None) -> str:
|
||||
"""基本持平;有抵消时接「,内部此消彼长:华东 +120、华北 -118」(最大的上升段、最大的下降段)。"""
|
||||
head = say(label, "基本持平") + (f"({pct(pct_total)})" if pct_total is not None else "")
|
||||
if offset is None:
|
||||
return head
|
||||
up_key, up_change, down_key, down_change = offset
|
||||
return head + say(",内部此消彼长:", up_key, f" {up_change}、", down_key, f" {down_change}")
|
||||
|
||||
|
||||
def decompose_unclosed(dims: str, gap: Any, *, ratio: bool) -> str:
|
||||
what = "各段合起来的比率和总数对不上" if ratio else "各段之和和总数对不上"
|
||||
return say("按", dims, f"拆开后{what}(差 {amount(abs(_q(gap)))}),不判断变化来自哪里")
|
||||
|
||||
|
||||
def ratio_mix(label: str, total: str, subject: str, w_base: Any, w_target: Any) -> str:
|
||||
return say(label, total, ",主要来自结构变化:", subject,
|
||||
f"占比从 {pct(w_base, signed=False)} 到 {pct(w_target, signed=False)}")
|
||||
|
||||
|
||||
def ratio_rate(label: str, total: str, subject: str) -> str:
|
||||
return say(label, total, ",主要来自各段自身的变化,", subject, "影响最大")
|
||||
|
||||
|
||||
def ratio_both(label: str, total: str, mix_share: Any, rate_share: Any) -> str:
|
||||
return say(label, total, f",结构变化和各段自身变化都有影响(各占 {pct(mix_share, signed=False)}、"
|
||||
f"{pct(rate_share, signed=False)})")
|
||||
|
||||
|
||||
def factor_driver(label: str, total: str, factor: str, contrib: Any, share: Any) -> str:
|
||||
return say(label, total, ",主要来自", factor, f"的变化(贡献 {amount(contrib, signed=True)},占 "
|
||||
f"{pct(share, signed=False)})")
|
||||
|
||||
|
||||
def factor_both(label: str, total: str, parts: Sequence[tuple[str, Any]]) -> str:
|
||||
"""乘积拆解里没有哪个因子占到「主要」:列出各因子占的比重。"""
|
||||
shown = "、".join(say(name, f"占 {pct(share, signed=False)}") for name, share in parts)
|
||||
return say(label, total, f",几个因子都有影响({shown})")
|
||||
|
||||
|
||||
def band_text(lo: Any, hi: Any, *, basis: str) -> str:
|
||||
fmt = (lambda v: pct(v)) if basis == "change" else (lambda v: amount(v))
|
||||
return f"{fmt(lo)} 至 {fmt(hi)}"
|
||||
|
||||
|
||||
def current_text(x: Any, *, basis: str, lag: int, grain: str) -> str:
|
||||
"""本期:变化率(和比较期比)或值本身。日数据 lag 7 写「比上周同一天」,lag 52 周 / 12 月 / 4 季写「同比」。"""
|
||||
if basis == "level":
|
||||
return f"本期为 {amount(x)}"
|
||||
if lag == 1:
|
||||
lead = "本期变化"
|
||||
elif grain == "day" and lag == 7:
|
||||
lead = "比上周同一天变化"
|
||||
elif (grain, lag) in (("week", 52), ("month", 12), ("quarter", 4)):
|
||||
lead = "同比变化"
|
||||
else:
|
||||
lead = f"比 {lag} 期前变化"
|
||||
return f"{lead} {pct(x)}"
|
||||
|
||||
|
||||
def baseline_phrase(verdict: str, label: str, current: str, n: int, unit: str, band: str) -> str:
|
||||
where = {"within_band": "在过去 {n} {u}的常规波动范围内", "outside_band": "超出过去 {n} {u}的常规波动范围",
|
||||
"far_outside_band": "明显超出过去 {n} {u}的常规波动范围"}[verdict].format(n=n, u=unit)
|
||||
return say(label, current, f",{where}({band})")
|
||||
|
||||
|
||||
def baseline_flat_phrase(verdict: str, label: str, current: str, n: int, unit: str, per: str, value: str) -> str:
|
||||
"""历史每期都一样(稳健标准差为 0)。"""
|
||||
if verdict == "within_band":
|
||||
return say(label, current, f",和过去 {n} {unit}一样(每{per}都是 {value})")
|
||||
return say(label, current, f",明显超出过去 {n} {unit}的常规波动范围(每{per}都是 {value})")
|
||||
|
||||
|
||||
def trend_phrase(verdict: str, label: str, n: int, unit: str, per: str, slope: str | None,
|
||||
run: tuple[int, str] | None) -> str:
|
||||
if verdict == "no_trend":
|
||||
return say(f"过去 {n} {unit}", label, "没有明显的上升或下降趋势")
|
||||
word = "上升" if verdict == "rising" else "下降"
|
||||
text = say(f"过去 {n} {unit}", label, f"呈{word}趋势") + (f"(每{per}约 {slope})" if slope else "")
|
||||
if run is not None:
|
||||
length, direction = run
|
||||
text += f",已连续 {length} {unit}{'上升' if direction == 'up' else '下降'}"
|
||||
return text
|
||||
|
||||
|
||||
def differ_pair(a: str, b: str, label: str, va: str, vb: str, higher: bool) -> str:
|
||||
"""一对的差异站得住:西南的退款率(5.0%)高于华东(2.1%)。"""
|
||||
return say(a, "的", label, f"({va}){'高于' if higher else '低于'}", b, f"({vb})")
|
||||
|
||||
|
||||
def differ_phrase(pairs: Sequence[str], total: int) -> str:
|
||||
"""差异站得住。比了几对时写「比了 4 对,2 对差异站得住:…」。"""
|
||||
if total == 1:
|
||||
return f"{pairs[0]},差异站得住"
|
||||
shown = ";".join(pairs[:2]) + ("等" if len(pairs) > 2 else "")
|
||||
return f"比了 {total} 对,{len(pairs)} 对差异站得住:{shown}"
|
||||
|
||||
|
||||
def no_evidence_phrase(a: str | None, b: str | None, label: str, sizes: tuple[str, str] | None, total: int) -> str:
|
||||
if total == 1 and a is not None and b is not None and sizes is not None:
|
||||
return say(a, "和", b, "的", label, f"差异可能只是样本波动(样本量 {sizes[0]}、{sizes[1]})")
|
||||
return say(f"比了 {total} 对,", label, "的差异可能都只是样本波动")
|
||||
|
||||
|
||||
def comparable_phrase(verdict: str, reasons: Sequence[str], unknown: Sequence[str]) -> str:
|
||||
if verdict == "incomparable":
|
||||
return "两期不可比:" + ";".join(reasons)
|
||||
if verdict == "conditional":
|
||||
return "两期有条件可比:" + ";".join(reasons)
|
||||
return "两期可比" + (f",{'、'.join(unknown)}核对不了" if unknown else "")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 判定标准(criterion):照实际阈值写的一句
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def criterion_additive(s: Mapping[str, Any], *, top_k_max: int | None, summed: bool) -> str:
|
||||
k = f"最多前 {top_k_max} 段" if top_k_max else "前几段"
|
||||
text = say("一段占总变化的", share_words(s["main_share"]), "以上算主要来源,", k, "合计占",
|
||||
share_words(s["concentration"]), "以上算集中,总变化不到基期的", threshold_pct(s["flat_pct"]),
|
||||
"算基本持平")
|
||||
return text + (",以各段之和为总数" if summed else ",各段之和要和总数对得上才判断变化来自哪里")
|
||||
|
||||
|
||||
def criterion_ratio(s: Mapping[str, Any]) -> str:
|
||||
return say("结构变化或各段自身变化占比率变化的", share_words(s["main_share"]), "以上算主要来自它,比率变化不到基期的",
|
||||
threshold_pct(s["flat_pct"]), f"算基本持平,分母不到 {number(s['min_denominator'])} 的段不参与多数判断")
|
||||
|
||||
|
||||
def criterion_factor(s: Mapping[str, Any]) -> str:
|
||||
return say("按对数平均拆分(LMDI),一个因子的贡献占总变化的", share_words(s["main_share"]),
|
||||
"以上算主要来自它,总变化不到基期的", threshold_pct(s["flat_pct"]), "算基本持平")
|
||||
|
||||
|
||||
def criterion_baseline(s: Mapping[str, Any], n: int | None, unit: str) -> str:
|
||||
span = f"过去 {n} {unit}" if n else f"过去最多 {number(s['max_history'])} 期"
|
||||
warn, alert = number(s["k_warn"]), number(s["k_alert"])
|
||||
return (f"本期和{span}的中位数比,差不到 {warn} 倍稳健标准差(1.4826 倍的中位绝对偏差)算常规波动,{warn} 到 "
|
||||
f"{alert} 倍算超出,{alert} 倍以上算明显超出;历史至少要 {number(s['min_history'])} 期")
|
||||
|
||||
|
||||
def criterion_trend(s: Mapping[str, Any], n: int | None, unit: str) -> str:
|
||||
span = f"最后 {n} {unit}" if n else f"最后最多 {number(s['max_history'])} 期"
|
||||
return (f"用{span}(至少 {number(s['trend_min_points'])} 期):Mann–Kendall 检验的双侧 p 值小于 "
|
||||
f"{number(s['trend_alpha'])} 时按 Theil–Sen 斜率的正负判上升或下降,否则判没有明显趋势")
|
||||
|
||||
|
||||
def criterion_differ(s: Mapping[str, Any], mode: str) -> str:
|
||||
alpha = number(s["differ_alpha"])
|
||||
if mode == "rate":
|
||||
return (f"双侧 p 值(比了几对时按 Holm 校正)小于 {alpha} 算差异站得住;有格子的期望频数不到 "
|
||||
f"{number(s['min_expected'])} 时用 Fisher 精确检验,否则用合并比例的 z 检验")
|
||||
return (f"Welch t 检验的双侧 p 值(比了几对时按 Holm 校正)小于 {alpha} 算差异站得住;一组样本数不到 "
|
||||
f"{number(s['min_n'])} 算样本少")
|
||||
|
||||
|
||||
CRITERION_COMPARABLE = ("只差日期、窗口一样长、是整期、星期构成相同、两边查询都没有数据问题才算可比;只有提醒项算有条件"
|
||||
"可比,有一项不通过算不可比;核对不了的项不判不可比")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CRITERION_COMPARABLE", "EMPTY_KEY", "HEADLINE_BASIS", "KEY_JOIN", "LABEL_MAX", "MEASURE_WORDS", "OP_NAMES",
|
||||
"PHRASE_CAVEATS", "PHRASE_ORDER", "UNVERIFIED_COMPLETE", "WEEKDAY_EFFECT", "amount", "band_text",
|
||||
"baseline_flat_phrase", "baseline_phrase", "caveat", "change", "clean_label", "comparable_phrase",
|
||||
"criterion_additive", "criterion_baseline", "criterion_differ", "criterion_factor", "criterion_ratio",
|
||||
"criterion_trend", "current_text", "decompose_concentrated", "decompose_dispersed", "decompose_driver",
|
||||
"decompose_flat", "decompose_unclosed", "differ_pair", "differ_phrase", "dims_text", "factor_both",
|
||||
"factor_driver", "key_text", "multiple_text", "names", "no_evidence_phrase", "null_periods_text",
|
||||
"null_segments_text", "number", "offset_text", "outlier_text", "pct", "pick_label", "points", "ratio_both",
|
||||
"ratio_change", "ratio_mix", "ratio_rate", "ratio_value", "record_text", "refused", "say", "share_words",
|
||||
"simpson_text", "small_denominator_text", "threshold_pct", "small_sample_mean_text", "small_sample_rate_text", "trend_phrase",
|
||||
"unconfirmed_text", "unlabeled_text", "with_caveats",
|
||||
]
|
||||
@@ -0,0 +1,254 @@
|
||||
"""可比性检查的现查(细案 01 §1「只读快照、不发查询」的唯一例外,§6):表里这一列的实际日期范围、按运行当天现算的界。
|
||||
|
||||
算子本身不查库。要现查的只有两样,都走 data/coverage 现成的只读探查(只读守卫、遮罩列不探、每条 PROBE_TIMEOUT_S 秒、总
|
||||
时限、同一张表 5 分钟内不重查、现查不成退回剖析事实):
|
||||
|
||||
- 范围:(数据源名, 表, 列) → contract.Probe {source, table, column, min, max, basis, at},min / max 是 ISO 日期;表里这一列
|
||||
一个值都没有时 min、max 都为 None(basis live)。basis 是 live(现查)/ profile(按剖析)/ observed(按取值观察),at 是现查的
|
||||
时刻(服务端时区,ISO,秒)或剖析、观察的日期。
|
||||
- 界:SQL 里按运行当天现算的界(CURRENT_DATE、DATE_SUB(…)、date('now', …))在同一个库里算出来:(数据源名, 界的 SQL 写法)
|
||||
→ Probe {source, expr, value, basis: live, at},value 是库里算出的日期(带时刻的写成「YYYY-MM-DD HH:MM:SS」)。
|
||||
|
||||
**结论里记下用到的现查结果,复算用记下的值**:Book 是一次分析的现查簿。run 时算子第一遍只登记要查什么(拿到的都是「不
|
||||
知道」),入口查完再算一遍,直到不再要新的(算子在拿到范围之后才可能要界,最多三轮);复算时 Book 里装的是结论记下的
|
||||
probes,不现查。结论的 probes 只记最后一遍真用到、查得到的,按用到的先后——同样的输入、同样的现查结果,结论逐字节相同。
|
||||
|
||||
入口(agent 分析工具、分析卡、口径卡的可比性挂钩)用 make_prober 生成 contract.Prober 交进 context["probe"],见它的说明。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from app.analysis.contract import Probe
|
||||
from app.data.names import name_key
|
||||
|
||||
#: 算子在拿到现查结果之后可能还要新的(先要范围、再要界):run 最多这么多轮
|
||||
MAX_ROUNDS = 3
|
||||
|
||||
|
||||
def _range_key(source: str, table: str, column: str) -> tuple[str, str, str, str]:
|
||||
return ("range", source or "", name_key(str(table)), name_key(str(column)))
|
||||
|
||||
|
||||
def _bound_key(source: str, expr: str) -> tuple[str, str, str]:
|
||||
return ("bound", source or "", str(expr))
|
||||
|
||||
|
||||
def probe_key(probe: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
"""一条记下的现查结果的键(Book 按它查)。"""
|
||||
if probe.get("expr") is not None:
|
||||
return _bound_key(str(probe.get("source") or ""), str(probe["expr"]))
|
||||
return _range_key(str(probe.get("source") or ""), str(probe.get("table") or ""), str(probe.get("column") or ""))
|
||||
|
||||
|
||||
class Book:
|
||||
"""一次分析的现查簿。known:{键: Probe 或 None(查过、查不到)}。算子经 range / bound 取;asked 是按先后登记的键,missing
|
||||
是登记了、簿里还没有的(run 据此去查),used 是最后真用到、查得到的(写进结论的 probes)。"""
|
||||
|
||||
def __init__(self, known: Mapping[tuple[str, ...], Probe | None] | Iterable[Probe] = ()) -> None:
|
||||
if isinstance(known, Mapping):
|
||||
self._known: dict[tuple[str, ...], Probe | None] = dict(known)
|
||||
self._closed = False
|
||||
else:
|
||||
self._known = {probe_key(p): p for p in known}
|
||||
self._closed = True # 复算:记下的就是全部,没记的当查不到
|
||||
self.asked: list[tuple[str, ...]] = []
|
||||
|
||||
def _ask(self, key: tuple[str, ...]) -> Probe | None:
|
||||
if key not in self.asked:
|
||||
self.asked.append(key)
|
||||
return self._known.get(key)
|
||||
|
||||
def range(self, source: str, table: str, column: str) -> Probe | None:
|
||||
return self._ask(_range_key(source, table, column))
|
||||
|
||||
def bound(self, source: str, expr: str) -> Probe | None:
|
||||
return self._ask(_bound_key(source, expr))
|
||||
|
||||
@property
|
||||
def missing(self) -> list[tuple[str, ...]]:
|
||||
return [] if self._closed else [k for k in self.asked if k not in self._known]
|
||||
|
||||
@property
|
||||
def used(self) -> list[Probe]:
|
||||
return [p for p in (self._known.get(k) for k in self.asked) if p is not None]
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now().astimezone().replace(tzinfo=None).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
async def fetch(book: Book, context: Mapping[str, Any]) -> dict[tuple[str, ...], Probe | None]:
|
||||
"""把簿里登记了、还没有的现查一遍:先 context["probe"],查不到的范围退回 context["facts"](数据源名 → {表名: 剖析事实},
|
||||
或数据源名 → 取剖析事实的函数),都没有记 None。交回新查到的那几条。"""
|
||||
from app.data import coverage
|
||||
|
||||
prober = context.get("probe")
|
||||
facts = context.get("facts") or {}
|
||||
out: dict[tuple[str, ...], Probe | None] = {}
|
||||
for key in book.missing:
|
||||
found: Probe | None = None
|
||||
if key[0] == "range":
|
||||
_, source, table, column = key
|
||||
if prober is not None:
|
||||
try:
|
||||
found = await prober(source, table, column)
|
||||
except Exception: # noqa: BLE001 - 探查失败就当不知道
|
||||
found = None
|
||||
if found is None:
|
||||
found = await _from_facts(facts.get(source), source, table, column, coverage)
|
||||
else:
|
||||
_, source, expr = key
|
||||
bounds = getattr(prober, "bounds", None)
|
||||
if bounds is not None:
|
||||
try:
|
||||
found = await bounds(source, expr)
|
||||
except Exception: # noqa: BLE001
|
||||
found = None
|
||||
out[key] = found
|
||||
return out
|
||||
|
||||
|
||||
async def _from_facts(facts: Any, source: str, table: str, column: str, coverage: Any) -> Probe | None:
|
||||
"""探查不成时按剖析事实兜底:facts 是 {表名: last_facts 的那一项} 或取它的函数(coverage.Facts)。"""
|
||||
found: Any = None
|
||||
try:
|
||||
if callable(facts):
|
||||
found = await facts(table)
|
||||
elif isinstance(facts, Mapping):
|
||||
found = next((v for k, v in facts.items() if name_key(str(k)) == name_key(table)
|
||||
or name_key(str(k).rsplit(".", 1)[-1]) == name_key(table.rsplit(".", 1)[-1])), None)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
data = coverage.facts_range(found, column)
|
||||
return _range_probe(source, table, column, data) if data else None
|
||||
|
||||
|
||||
def _range_probe(source: str, table: str, column: str, data: Mapping[str, Any]) -> Probe:
|
||||
basis = data.get("basis") if data.get("basis") in ("live", "profile", "observed") else "live"
|
||||
at = _now() if basis == "live" else str(data.get("at") or "")
|
||||
return {"source": source, "table": table, "column": column, "min": data.get("min"), "max": data.get("max"),
|
||||
"basis": basis, "at": at} # type: ignore[typeddict-item]
|
||||
|
||||
|
||||
class LiveProber:
|
||||
"""contract.Prober:await prober(数据源名, 表, 列) → Probe | None;另有 await prober.bounds(数据源名, 界的 SQL) 在库里算界,
|
||||
prober.dialect(数据源名) 是这个源的 sqlglot 方言(算子按它读 SQL、写界)。见 make_prober。"""
|
||||
|
||||
def __init__(self, sources: Mapping[str, Any], *, masked: Mapping[str, Iterable[str]] | None,
|
||||
facts: Mapping[str, Callable[[str], Awaitable[Mapping[str, Any] | None]]] | None,
|
||||
cache: Mapping[str, Any] | None, budget: float,
|
||||
clock: Callable[[], str] | None) -> None:
|
||||
from app.data import coverage
|
||||
|
||||
self._sources = dict(sources)
|
||||
self._masked = {k: {str(c).strip().lower() for c in v} for k, v in (masked or {}).items()}
|
||||
self._facts = dict(facts or {})
|
||||
self._cache: dict[str, Any] = dict(cache or {})
|
||||
self._budget = budget
|
||||
self._deadline: float | None = None
|
||||
self._clock = clock or _now
|
||||
self._bounds: dict[tuple[str, str], Any] = {}
|
||||
self._coverage = coverage
|
||||
|
||||
def dialect(self, source: str) -> str | None:
|
||||
from app.data.sqlcheck import dialect_of
|
||||
|
||||
found = self._sources.get(source)
|
||||
return dialect_of(getattr(found, "kind", None)) if found is not None else None
|
||||
|
||||
def _expired(self) -> bool:
|
||||
if self._deadline is None:
|
||||
self._deadline = time.monotonic() + self._budget
|
||||
return time.monotonic() > self._deadline
|
||||
|
||||
def _hidden(self, source: Any, name: str) -> set[str]:
|
||||
if name in self._masked:
|
||||
return self._masked[name]
|
||||
from app.data.engine import masked_columns
|
||||
|
||||
return {c.lower() for c in masked_columns(getattr(source, "options", None) or {})}
|
||||
|
||||
def _facts_of(self, name: str, source: Any) -> Callable[[str], Awaitable[Mapping[str, Any] | None]]:
|
||||
return self._facts.get(name) or profile_facts(source)
|
||||
|
||||
async def __call__(self, source: str, table: str, column: str) -> Probe | None:
|
||||
found = self._sources.get(source)
|
||||
if found is None or str(column).strip().lower() in self._hidden(found, source):
|
||||
return None
|
||||
if self._expired():
|
||||
return None
|
||||
cache = self._cache.setdefault(source, self._coverage.Cache())
|
||||
data = await self._coverage.column_range(found, table, column, facts=self._facts_of(source, found), cache=cache)
|
||||
if data is None:
|
||||
return None
|
||||
probe = _range_probe(source, table, column, data)
|
||||
if probe["basis"] == "live":
|
||||
probe["at"] = self._clock()
|
||||
return probe
|
||||
|
||||
async def bounds(self, source: str, expr: str) -> Probe | None:
|
||||
found = self._sources.get(source)
|
||||
if found is None or self._expired():
|
||||
return None
|
||||
key = (source, expr)
|
||||
if key not in self._bounds:
|
||||
values = await self._coverage.bound_values(found, [expr])
|
||||
self._bounds[key] = values[0] if values else None
|
||||
value = self._bounds[key]
|
||||
if value is None:
|
||||
return None
|
||||
return {"source": source, "expr": expr, "value": str(value), "basis": "live", # type: ignore[typeddict-unknown-key]
|
||||
"at": self._clock()}
|
||||
|
||||
|
||||
def profile_facts(source: Any) -> Callable[[str], Awaitable[Mapping[str, Any] | None]]:
|
||||
"""这个数据源上一张表的剖析事实(catalog_profile_jobs.last_facts 的那一项,同 tools/datasource._coverage 的退回):上传
|
||||
表格的快照对不上这次读的那一版时不用。"""
|
||||
async def facts(table: str) -> Mapping[str, Any] | None:
|
||||
from app.data import catalog as data_catalog
|
||||
from app.data.catalog_profile_jobs import last_facts
|
||||
from app.db.base import SessionLocal
|
||||
|
||||
source_id = getattr(source, "id", None)
|
||||
if not source_id:
|
||||
return None
|
||||
name = data_catalog.resolve_table_name(getattr(source, "schema_cache", None), table) or table
|
||||
async with SessionLocal() as session:
|
||||
found = (await last_facts(session, str(source_id), [name])).get(name)
|
||||
snapshot = getattr(source, "snapshot_id", None)
|
||||
if found and snapshot and found.get("snapshot_id") != snapshot:
|
||||
return None
|
||||
return found
|
||||
|
||||
return facts
|
||||
|
||||
|
||||
def make_prober(sources: Mapping[str, Any], *, masked: Mapping[str, Iterable[str]] | None = None,
|
||||
facts: Mapping[str, Callable[[str], Awaitable[Mapping[str, Any] | None]]] | None = None,
|
||||
cache: Mapping[str, Any] | None = None, budget: float | None = None,
|
||||
clock: Callable[[], str] | None = None) -> LiveProber:
|
||||
"""生成一次分析调用用的探查器(contract.Prober,交进 context["probe"])。
|
||||
|
||||
- sources:数据源名 → 数据源对象(DataSource 行,或上传源绑定快照的视图:同 tools/datasource 建工具时的 source,要有
|
||||
name、kind、options、schema_cache;上传源另有 snapshot_id)。名字和快照里的 source 一致。没给的源不探。
|
||||
- masked:数据源名 → 遮罩列(不分大小写)。缺省按数据源 options 里设的遮罩列(data/engine.masked_columns)。遮罩列上
|
||||
不探,探查器直接交回 None。
|
||||
- facts:数据源名 → 取剖析事实的函数(coverage.Facts:表名 → last_facts 的那一项)。缺省按 profile_facts 现读库(同查询
|
||||
工具的覆盖检查)。
|
||||
- cache:数据源名 → coverage.Cache(同一张表、同一个界一次运行里只查一次);不给就每个探查器各自一份。另外 coverage
|
||||
在进程里把现查到的范围记 5 分钟。
|
||||
- budget:这个探查器所有探查加起来最多多少秒(缺省 coverage.BUDGET_S,从第一次探查算起);过了的一律交回 None。一次
|
||||
分析调用建一个。
|
||||
- clock:现查时刻的写法(测试用);缺省是服务端时区的 ISO 时间(到秒)。
|
||||
"""
|
||||
from app.data import coverage
|
||||
|
||||
return LiveProber(sources, masked=masked, facts=facts, cache=cache,
|
||||
budget=coverage.BUDGET_S if budget is None else budget, clock=clock)
|
||||
|
||||
|
||||
__all__ = ["Book", "LiveProber", "MAX_ROUNDS", "fetch", "make_prober", "probe_key", "profile_facts"]
|
||||
@@ -0,0 +1,300 @@
|
||||
"""放回历史看(细案 01 §3、§4):波动基线(这个变化大不大、算不算异常)和趋势(是持续在变,还是一次波动)。
|
||||
|
||||
**参数**:query(一份快照:一条序列;要看某一段就在 SQL 里筛好)、period(期次列)、measure(度量列);可选 grain(day /
|
||||
week / month / quarter,缺省按期次值推断,见 analysis/periods)、label。波动基线另有 target(本期,缺省最后一期)、basis
|
||||
(change 缺省:和比较期比的变化率;level:值本身,给存量、比率用)、lag(和往前第几期比,缺省日数据 7、其余 1;同比写 52 /
|
||||
12 / 4,历史要相应够长)。
|
||||
|
||||
**前提**(不满足就算不了,原因照实写):期次认得出(period_unreadable)、本期完整(同可比性检查的整期:没过完、数据没到、
|
||||
SQL 截掉一部分 → partial_target;表的日期范围查不到时只按运行当天判,附一句核对不了)、本期和比较期之间不缺期(target_gap)、
|
||||
比较期不为 0(zero_base)、历史够长(history_short);趋势要最后几期连着(中间缺期 series_gap)。度量格为空的那一期跳过。
|
||||
|
||||
**波动基线**:历史的中位数 m、中位绝对偏差 MAD,σ̂ = 1.4826·MAD,稳健 z = (x − m)/σ̂;常规波动范围 m ± k_warn·σ̂;全程
|
||||
精确的有理数。σ̂ = 0(历史每期都一样)时本期一样判常规、不一样判明显超出。本期的值(不论 basis)是历史各期里最高 / 最低的
|
||||
加 record_high / record_low:「历史新高」说的是值本身。为什么不用均值和标准差:促销周这类极端值会把它们拉宽。
|
||||
|
||||
**趋势**:最后 n 期(期次连着的那一段,trend_min_points ≤ n ≤ max_history + 1)的 Theil–Sen 斜率、Mann–Kendall 双侧 p,
|
||||
相对斜率 = 斜率 / |序列中位数|。p < trend_alpha 时按斜率的正负判上升、下降。末尾连续同向的期数从最后一期往前数(变化为 0
|
||||
断开)。已知局限:Mann–Kendall 假设各期独立,业务序列常有自相关,p 会偏小;所以缺省 n 至少 6、α 取 0.05,判定语只说
|
||||
「呈上升趋势」,不说「显著上升」。日数据附一句星期效应。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
from app.analysis import comparable as cmp
|
||||
from app.analysis import periods as pd
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis import stats
|
||||
from app.analysis.common import Env, build, column_label, label_for, need, refuse, sign_word, today_param
|
||||
from app.analysis.contract import CallError
|
||||
from app.analysis.inputs import Snap, same_value
|
||||
from app.analysis.numbers import canon, to_fraction
|
||||
|
||||
#: 稳健标准差 = 1.4826 × MAD(正态时等于标准差)
|
||||
MAD_SCALE = Fraction(14826, 10000)
|
||||
#: lag 的上限(周数据同比是 52)
|
||||
_LAG_MAX = 260
|
||||
|
||||
|
||||
@dataclass
|
||||
class Series:
|
||||
snap: Snap
|
||||
period: str
|
||||
measure: str
|
||||
grain: str
|
||||
label: str
|
||||
points: list[tuple[pd.Period, Fraction]] # 按期次排好、度量不为空的
|
||||
nulls: int
|
||||
params: dict[str, Any]
|
||||
|
||||
|
||||
def _load(raw: Mapping[str, Any], env: Env, used: list[str]) -> Series | Any:
|
||||
"""读出一条序列;算不了时直接交回 refused 的结论。"""
|
||||
snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
|
||||
period = snap.column(need(raw, "period", "期次列"), "期次列")
|
||||
measure = snap.column(need(raw, "measure", "度量列"), "度量列")
|
||||
grain = raw.get("grain")
|
||||
if grain is not None and grain not in (*pd.GRAINS, "seq"):
|
||||
raise CallError("bad_grain", "粒度只能写 day、week、month、quarter,或者不写(按期次值推断)")
|
||||
params: dict[str, Any] = {"query": snap.ref, "period": period, "measure": measure}
|
||||
label = label_for(env, raw.get("label"), snap, measure)
|
||||
if raw.get("label") is not None:
|
||||
params["label"] = ph.clean_label(raw.get("label")) or None
|
||||
criterion = _criterion(env, None, "期")
|
||||
env.inputs.use(snap, "data", [period, measure])
|
||||
if snap.truncated:
|
||||
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行,序列不完整",
|
||||
criterion=criterion)
|
||||
i_period = snap.index(period)
|
||||
values = [row[i_period] if i_period < len(row) else None for row in snap.rows]
|
||||
parsed = pd.parse(values, grain)
|
||||
if parsed is None:
|
||||
sample = "、".join(dict.fromkeys(ph.key_text([v]) for v in values[:3]))
|
||||
return refuse(env, params, used, "period_unreadable",
|
||||
f"期次列「{period}」的值认不出是哪一期({sample}):要日期、YYYY-MM、YYYY-Www、YYYY-Qn 或整数序号",
|
||||
criterion=criterion)
|
||||
items, grain = parsed
|
||||
params["grain"] = grain
|
||||
seen: set[int] = set()
|
||||
points: list[tuple[pd.Period, Fraction]] = []
|
||||
nulls = 0
|
||||
for r, item in enumerate(items):
|
||||
if item.ordinal in seen:
|
||||
raise CallError("duplicate_rows", ph.say(f"{snap.ref} 里同一期有不止一行:在 SQL 里按 ",
|
||||
column_label(env, snap, period), " 汇总,每期一行"))
|
||||
seen.add(item.ordinal)
|
||||
value = snap.number(r, measure, "度量列")
|
||||
if value is None:
|
||||
nulls += 1
|
||||
continue
|
||||
points.append((item, value))
|
||||
points.sort(key=lambda p: p[0].ordinal)
|
||||
return Series(snap, period, measure, grain, label, points, nulls, params)
|
||||
|
||||
|
||||
def _criterion(env: Env, n: int | None, unit: str) -> str:
|
||||
if env.op == "baseline":
|
||||
return ph.criterion_baseline(env.settings, n, unit)
|
||||
return ph.criterion_trend(env.settings, n, unit)
|
||||
|
||||
|
||||
def _completeness(env: Env, series: Series, item: pd.Period) -> tuple[str, str | None]:
|
||||
"""本期齐不齐(同可比性检查的整期):(pass / short / unknown, 不齐的那一句)。"""
|
||||
series.params["today"] = today_param(env)
|
||||
if item.start is None:
|
||||
return "unknown", None
|
||||
span = cmp.period_span("本期", series.snap, series.period, item, series.grain, series.measure, env.facts, env.book)
|
||||
return cmp.span_coverage(span, env.today, env.book) # type: ignore[arg-type]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 波动基线
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def baseline(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["k_warn", "k_alert", "min_history", "max_history"]
|
||||
loaded = _load(raw, env, used)
|
||||
if not isinstance(loaded, Series):
|
||||
return loaded
|
||||
series, params = loaded, loaded.params
|
||||
basis = raw.get("basis") or "change"
|
||||
if basis not in ("change", "level"):
|
||||
raise CallError("bad_basis", "比法只能写 change(和比较期比的变化率)或 level(值本身)")
|
||||
lag = raw.get("lag", pd.DEFAULT_LAG[series.grain])
|
||||
if isinstance(lag, bool) or not isinstance(lag, int) or not 1 <= lag <= _LAG_MAX:
|
||||
raise CallError("bad_lag", f"往前比几期要写 1 到 {_LAG_MAX} 的整数(同比:周数据 52、月数据 12、季度 4)")
|
||||
params.update(basis=basis, lag=lag)
|
||||
unit, per = pd.COUNT_WORD[series.grain], pd.PER_WORD[series.grain]
|
||||
criterion = _criterion(env, None, unit)
|
||||
if not series.points:
|
||||
return refuse(env, params, used, "history_short", f"只有 0 期历史,至少要 {int(env.settings['min_history'])} 期",
|
||||
criterion=criterion)
|
||||
if raw.get("target") is not None:
|
||||
found = next((p for p in series.points if same_value(raw["target"], p[0].raw)), None)
|
||||
if found is None:
|
||||
raise CallError("period_not_found", f"期次列「{series.period}」里没有「{raw['target']}」(或那一期为空):"
|
||||
"本期要写序列里有值的一期,不写就是最后一期")
|
||||
else:
|
||||
found = series.points[-1]
|
||||
target, x_level = found
|
||||
params["target"] = target.raw
|
||||
caveats: list[dict[str, str]] = []
|
||||
status, why = _completeness(env, series, target)
|
||||
if status == "short":
|
||||
return refuse(env, params, used, "partial_target", f"{why},和完整的{per}比没有意义", criterion=criterion)
|
||||
if status == "unknown" and target.start is not None:
|
||||
caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE))
|
||||
by_ordinal = {p.ordinal: v for p, v in series.points}
|
||||
before = [(p, v) for p, v in series.points if p.ordinal < target.ordinal]
|
||||
if basis == "change":
|
||||
prev = by_ordinal.get(target.ordinal - lag)
|
||||
if prev is None:
|
||||
return refuse(env, params, used, "target_gap",
|
||||
f"本期和它的比较期(往前第 {lag} {'期' if series.grain == 'seq' else unit})之间缺期,没有比较期的值",
|
||||
criterion=criterion)
|
||||
if prev == 0:
|
||||
return refuse(env, params, used, "zero_base", ph.say("比较期的", series.label, "为 0,算不了变化率"),
|
||||
criterion=criterion)
|
||||
x = (x_level - prev) / abs(prev)
|
||||
history = [(p, (v - by_ordinal[p.ordinal - lag]) / abs(by_ordinal[p.ordinal - lag]), v) for p, v in before
|
||||
if by_ordinal.get(p.ordinal - lag) not in (None, 0)]
|
||||
else:
|
||||
x = x_level
|
||||
history = [(p, v, v) for p, v in before]
|
||||
history = history[-int(env.settings["max_history"]):]
|
||||
n = len(history)
|
||||
criterion = _criterion(env, n, unit)
|
||||
if n < int(env.settings["min_history"]):
|
||||
return refuse(env, params, used, "history_short", f"只有 {n} 期历史,至少要 {int(env.settings['min_history'])} 期",
|
||||
criterion=criterion)
|
||||
hv = [h for _, h, _ in history]
|
||||
m = stats.median(hv)
|
||||
sigma = MAD_SCALE * stats.mad(hv, m)
|
||||
k_warn, k_alert = env.threshold("k_warn"), env.threshold("k_alert")
|
||||
if sigma == 0:
|
||||
z = None
|
||||
code = "within_band" if x == m else "far_outside_band"
|
||||
else:
|
||||
z = (x - m) / sigma
|
||||
code = "within_band" if abs(z) < k_warn else "outside_band" if abs(z) < k_alert else "far_outside_band"
|
||||
lo, hi = m - k_warn * sigma, m + k_warn * sigma
|
||||
levels = [lv for _, _, lv in history]
|
||||
if x_level > max(levels):
|
||||
caveats.append(ph.caveat("record_high", ph.record_text(True, series.label, n, unit)))
|
||||
elif x_level < min(levels):
|
||||
caveats.append(ph.caveat("record_low", ph.record_text(False, series.label, n, unit)))
|
||||
if series.nulls:
|
||||
caveats.append(ph.caveat("null_values", ph.null_periods_text(series.nulls, series.label)))
|
||||
current = ph.current_text(x, basis=basis, lag=lag, grain=series.grain)
|
||||
if sigma == 0:
|
||||
shown = ph.pct(m) if basis == "change" else ph.amount(m)
|
||||
main = ph.baseline_flat_phrase(code, series.label, current, n, unit, per, shown)
|
||||
else:
|
||||
main = ph.baseline_phrase(code, series.label, current, n, unit, ph.band_text(lo, hi, basis=basis))
|
||||
rows = [[p.raw, canon(lv), canon(h) if basis == "change" else None, "历史"] for p, h, lv in history]
|
||||
rows.append([target.raw, canon(x_level), canon(x) if basis == "change" else None, "本期"])
|
||||
table = {"columns": [series.period, "value", "change", "role"],
|
||||
"labels": {series.period: column_label(env, series.snap, series.period), "value": series.label,
|
||||
"change": "变化率", "role": "说明"},
|
||||
"rows": rows}
|
||||
values = {"x": canon(x), "value": canon(x_level), "median": canon(m), "band_lo": canon(lo), "band_hi": canon(hi),
|
||||
"z": canon(z) if z is not None else None, "rank": str(stats.rank_desc(x, hv)), "n_history": str(n),
|
||||
"hist_min": canon(min(hv)), "hist_max": canon(max(hv))}
|
||||
verdict = {"code": code, "subjects": [], "direction": sign_word(x - m)}
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
|
||||
phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 趋势
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_length(values: list[Fraction]) -> tuple[int, str | None]:
|
||||
"""末尾连续同向:从最后一期往前数,相邻变化同号的有几个(变化为 0 断开)。"""
|
||||
changes = [b - a for a, b in zip(values, values[1:])]
|
||||
if not changes or changes[-1] == 0:
|
||||
return 0, None
|
||||
sign = changes[-1] > 0
|
||||
count = 0
|
||||
for c in reversed(changes):
|
||||
if c == 0 or (c > 0) != sign:
|
||||
break
|
||||
count += 1
|
||||
return count, "up" if sign else "down"
|
||||
|
||||
|
||||
def trend(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["trend_min_points", "trend_alpha", "max_history"]
|
||||
loaded = _load(raw, env, used)
|
||||
if not isinstance(loaded, Series):
|
||||
return loaded
|
||||
series, params = loaded, loaded.params
|
||||
unit, per = pd.COUNT_WORD[series.grain], pd.PER_WORD[series.grain]
|
||||
minimum = int(env.settings["trend_min_points"])
|
||||
criterion = _criterion(env, None, unit)
|
||||
if not series.points:
|
||||
return refuse(env, params, used, "history_short", f"只有 0 期,至少要 {minimum} 期", criterion=criterion)
|
||||
last = series.points[-1][0]
|
||||
caveats: list[dict[str, str]] = []
|
||||
status, why = _completeness(env, series, last)
|
||||
if status == "short":
|
||||
return refuse(env, params, used, "partial_target", f"{why},和完整的{per}比没有意义", criterion=criterion)
|
||||
if status == "unknown" and last.start is not None:
|
||||
caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE))
|
||||
cap = int(env.settings["max_history"]) + 1
|
||||
tail = [series.points[-1]]
|
||||
for point in reversed(series.points[:-1]):
|
||||
if len(tail) >= cap or point[0].ordinal != tail[-1][0].ordinal - 1:
|
||||
break
|
||||
tail.append(point)
|
||||
tail.reverse()
|
||||
n = len(tail)
|
||||
criterion = _criterion(env, n, unit)
|
||||
if n < minimum:
|
||||
gap = len(series.points) > n and len(tail) < cap
|
||||
if gap:
|
||||
return refuse(env, params, used, "series_gap",
|
||||
f"最后连着的只有 {n} 期,再往前缺期(至少要连着的 {minimum} 期)", criterion=criterion)
|
||||
return refuse(env, params, used, "history_short", f"只有 {n} 期,至少要 {minimum} 期", criterion=criterion)
|
||||
ys = [v for _, v in tail]
|
||||
xs = [Fraction(i) for i in range(n)]
|
||||
slope = stats.theil_sen(xs, ys)
|
||||
med = stats.median(ys)
|
||||
rel = slope / abs(med) if med != 0 else None
|
||||
mk = stats.mann_kendall(ys)
|
||||
alpha = env.threshold("trend_alpha")
|
||||
p = mk.p if isinstance(mk.p, Fraction) else to_fraction(mk.p)
|
||||
if p is not None and p < alpha and slope > 0:
|
||||
code = "rising"
|
||||
elif p is not None and p < alpha and slope < 0:
|
||||
code = "falling"
|
||||
else:
|
||||
code = "no_trend"
|
||||
run, direction = run_length(ys)
|
||||
if series.grain == "day":
|
||||
caveats.append(ph.caveat("weekday_effect", ph.WEEKDAY_EFFECT))
|
||||
if series.nulls:
|
||||
caveats.append(ph.caveat("null_values", ph.null_periods_text(series.nulls, series.label)))
|
||||
slope_text = ph.pct(rel) if rel is not None else ph.amount(slope, signed=True)
|
||||
main = ph.trend_phrase(code, series.label, n, unit, per, slope_text,
|
||||
(run, direction) if run >= 3 and direction else None) # type: ignore[arg-type]
|
||||
rows = [[p_.raw, canon(v), canon(v - tail[i - 1][1]) if i else None] for i, (p_, v) in enumerate(tail)]
|
||||
table = {"columns": [series.period, "value", "change"],
|
||||
"labels": {series.period: column_label(env, series.snap, series.period), "value": series.label,
|
||||
"change": "比上一期"},
|
||||
"rows": rows}
|
||||
values = {"slope": canon(slope), "rel_slope": canon(rel) if rel is not None else None, "p": canon(mk.p),
|
||||
"s": str(mk.s), "n": str(n), "run_length": str(run), "run_direction": direction}
|
||||
verdict = {"code": code, "subjects": [], "direction": "up" if code == "rising" else "down" if code == "falling"
|
||||
else None}
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
|
||||
phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats)
|
||||
|
||||
|
||||
__all__ = ["MAD_SCALE", "baseline", "run_length", "trend"]
|
||||
@@ -0,0 +1,467 @@
|
||||
"""分析用的统计函数(docs/design/analysis-round/01-算子与判定.md §3–§5,总方案 D13):纯 Python,不加依赖、不用 numpy。
|
||||
|
||||
为什么不装 scipy:几十 MB 的依赖只为几个函数,本机装包慢;评测本来就有一套纯 Python 的统计(Wilson 区间、精确 McNemar),
|
||||
两边要是同一套。所以 Wilson 从 eval/stats 搬到这里,评测改为引用(行为不变)。每个函数都用公认的表值钉住
|
||||
(tests/test_analysis_stats.py:正态 0.975 分位 1.959964、t(10) 双侧 0.05 临界值 2.228139、Fisher 品茶表双侧 0.4857……)。
|
||||
|
||||
- 正态:分布函数用 math.erfc;分位数用 Acklam 的有理逼近再按分布函数做两步 Halley 修正(到浮点精度)。
|
||||
- t 分布:分布函数用正则化不完全 β 函数(连分式,Lentz 法);自由度很大(> 1e7)时和正态差在 1e-7 以内,直接按正态算;
|
||||
分位数按分布函数二分求逆。
|
||||
- 比率:Wilson 区间(缺省按评测的写法四舍五入到 4 位,places=None 不取整)、两比率之差的 Newcombe 区间(混合 score
|
||||
法,即 Newcombe 1998 的方法 10)、合并比例的 z 检验、Fisher 精确检验(超几何概率用 math.lgamma 累加,双侧按「不比
|
||||
观测到的那张表更可能」的表求和,比较时留 1e-7 的相对余量,同 R 和 scipy)。
|
||||
- 均值:Welch t 检验,自由度按 Welch–Satterthwaite。
|
||||
- 多重比较:Holm 逐步校正。
|
||||
- 趋势:Mann–Kendall(n ≤ 10 且没有并列时用精确分布:S 的分布由逆序数的分布递推算出,不查表;否则正态近似,方差按
|
||||
并列修正,带连续性校正)、Theil–Sen 斜率(两两斜率的中位数)。
|
||||
- 中位数、中位绝对偏差、名次。
|
||||
|
||||
能精确算的照精确算:中位数、MAD、Theil–Sen 对 Fraction 输入给 Fraction,Mann–Kendall 的精确 p 是 Fraction。浮点只出现在
|
||||
要开方、取对数、查分布的地方,结论里的数再按 analysis/numbers 统一成规范的十进制字符串。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections import Counter
|
||||
from collections.abc import Sequence
|
||||
from fractions import Fraction
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
#: 95% 区间的 z(正态 0.975 分位)
|
||||
Z95 = 1.959963984540054
|
||||
_SQRT2 = math.sqrt(2.0)
|
||||
#: t 分布自由度大到这个数以上时按正态算(相对误差约 1/自由度)
|
||||
_T_AS_NORMAL_DF = 1e7
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 正态分布
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def normal_cdf(x: float) -> float:
|
||||
"""标准正态分布函数 Φ(x)。"""
|
||||
return 0.5 * math.erfc(-x / _SQRT2)
|
||||
|
||||
|
||||
def normal_two_sided(z: float) -> float:
|
||||
"""标准正态的双侧尾概率 P(|Z| ≥ |z|)。"""
|
||||
return math.erfc(abs(z) / _SQRT2)
|
||||
|
||||
|
||||
_A = (-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, 1.383577518672690e+02,
|
||||
-3.066479806614716e+01, 2.506628277459239e+00)
|
||||
_B = (-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, 6.680131188771972e+01,
|
||||
-1.328068155288572e+01)
|
||||
_C = (-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, -2.549732539343734e+00,
|
||||
4.374664141464968e+00, 2.938163982698783e+00)
|
||||
_D = (7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, 3.754408661907416e+00)
|
||||
|
||||
|
||||
def normal_ppf(p: float) -> float:
|
||||
"""标准正态的 p 分位数 Φ⁻¹(p)。p ≤ 0 为负无穷,p ≥ 1 为正无穷。"""
|
||||
if p <= 0.0:
|
||||
return -math.inf
|
||||
if p >= 1.0:
|
||||
return math.inf
|
||||
low = 0.02425
|
||||
if p < low:
|
||||
q = math.sqrt(-2 * math.log(p))
|
||||
x = (((((_C[0] * q + _C[1]) * q + _C[2]) * q + _C[3]) * q + _C[4]) * q + _C[5]) / \
|
||||
((((_D[0] * q + _D[1]) * q + _D[2]) * q + _D[3]) * q + 1)
|
||||
elif p <= 1 - low:
|
||||
q = p - 0.5
|
||||
r = q * q
|
||||
x = (((((_A[0] * r + _A[1]) * r + _A[2]) * r + _A[3]) * r + _A[4]) * r + _A[5]) * q / \
|
||||
(((((_B[0] * r + _B[1]) * r + _B[2]) * r + _B[3]) * r + _B[4]) * r + 1)
|
||||
else:
|
||||
q = math.sqrt(-2 * math.log1p(-p))
|
||||
x = -(((((_C[0] * q + _C[1]) * q + _C[2]) * q + _C[3]) * q + _C[4]) * q + _C[5]) / \
|
||||
((((_D[0] * q + _D[1]) * q + _D[2]) * q + _D[3]) * q + 1)
|
||||
for _ in range(2):
|
||||
# Halley 修正:误差按分布函数现算,两步就到浮点精度
|
||||
e = (normal_cdf(x) - p) if p < 0.5 else -(0.5 * math.erfc(x / _SQRT2) - (1 - p))
|
||||
u = e * math.sqrt(2 * math.pi) * math.exp(x * x / 2)
|
||||
x = x - u / (1 + x * u / 2)
|
||||
return x
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 正则化不完全 β 函数与 t 分布
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_CF_MAX_ITER = 20_000
|
||||
_CF_EPS = 1e-15
|
||||
_CF_TINY = 1e-300
|
||||
|
||||
|
||||
def _beta_cf(a: float, b: float, x: float) -> float:
|
||||
"""不完全 β 函数的连分式(Lentz 法,Numerical Recipes 的 betacf)。"""
|
||||
qab, qap, qam = a + b, a + 1.0, a - 1.0
|
||||
c = 1.0
|
||||
d = 1.0 - qab * x / qap
|
||||
d = 1.0 / (d if abs(d) >= _CF_TINY else _CF_TINY)
|
||||
h = d
|
||||
for m in range(1, _CF_MAX_ITER + 1):
|
||||
m2 = 2 * m
|
||||
aa = m * (b - m) * x / ((qam + m2) * (a + m2))
|
||||
d = 1.0 + aa * d
|
||||
d = 1.0 / (d if abs(d) >= _CF_TINY else _CF_TINY)
|
||||
c = 1.0 + aa / c
|
||||
c = c if abs(c) >= _CF_TINY else _CF_TINY
|
||||
h *= d * c
|
||||
aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2))
|
||||
d = 1.0 + aa * d
|
||||
d = 1.0 / (d if abs(d) >= _CF_TINY else _CF_TINY)
|
||||
c = 1.0 + aa / c
|
||||
c = c if abs(c) >= _CF_TINY else _CF_TINY
|
||||
step = d * c
|
||||
h *= step
|
||||
if abs(step - 1.0) < _CF_EPS:
|
||||
break
|
||||
return h
|
||||
|
||||
|
||||
def betainc(a: float, b: float, x: float) -> float:
|
||||
"""正则化不完全 β 函数 I_x(a, b),a、b > 0,0 ≤ x ≤ 1。"""
|
||||
if x <= 0.0:
|
||||
return 0.0
|
||||
if x >= 1.0:
|
||||
return 1.0
|
||||
log_front = math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b) + a * math.log(x) + b * math.log1p(-x)
|
||||
if x < (a + 1.0) / (a + b + 2.0):
|
||||
return math.exp(log_front) * _beta_cf(a, b, x) / a
|
||||
return 1.0 - math.exp(log_front) * _beta_cf(b, a, 1.0 - x) / b
|
||||
|
||||
|
||||
def t_two_sided(t: float, df: float) -> float:
|
||||
"""自由度 df 的 t 分布的双侧尾概率 P(|T| ≥ |t|)。"""
|
||||
if math.isinf(t):
|
||||
return 0.0
|
||||
if df > _T_AS_NORMAL_DF or math.isinf(df):
|
||||
return normal_two_sided(t)
|
||||
return min(1.0, betainc(df / 2.0, 0.5, df / (df + t * t)))
|
||||
|
||||
|
||||
def t_cdf(t: float, df: float) -> float:
|
||||
"""自由度 df 的 t 分布的分布函数。"""
|
||||
tail = t_two_sided(t, df) / 2.0
|
||||
return 1.0 - tail if t >= 0 else tail
|
||||
|
||||
|
||||
def t_ppf(p: float, df: float) -> float:
|
||||
"""自由度 df 的 t 分布的 p 分位数:按分布函数二分求逆。"""
|
||||
if p <= 0.0:
|
||||
return -math.inf
|
||||
if p >= 1.0:
|
||||
return math.inf
|
||||
if p == 0.5:
|
||||
return 0.0
|
||||
if df > _T_AS_NORMAL_DF or math.isinf(df):
|
||||
return normal_ppf(p)
|
||||
target = min(p, 1.0 - p)
|
||||
lo, hi = -1.0, 0.0
|
||||
while t_cdf(lo, df) > target:
|
||||
lo, hi = lo * 2.0, lo
|
||||
if lo < -1e12:
|
||||
break
|
||||
for _ in range(200):
|
||||
mid = (lo + hi) / 2.0
|
||||
if t_cdf(mid, df) > target:
|
||||
hi = mid
|
||||
else:
|
||||
lo = mid
|
||||
if hi - lo <= 1e-15 * max(1.0, abs(mid)):
|
||||
break
|
||||
x = (lo + hi) / 2.0
|
||||
return x if p < 0.5 else -x
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 比率
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def wilson(passed: int, n: int, z: float = Z95, *, places: int | None = 4) -> tuple[float, float] | None:
|
||||
"""Wilson 区间(缺省 95%)。n 为 0 时没有区间。places 是四舍五入的位数(缺省 4 位,评测一直这样显示);None 不取整。"""
|
||||
if n <= 0:
|
||||
return None
|
||||
p = passed / n
|
||||
denom = 1 + z * z / n
|
||||
centre = (p + z * z / (2 * n)) / denom
|
||||
half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom
|
||||
low, high = max(0.0, centre - half), min(1.0, centre + half)
|
||||
if places is None:
|
||||
return low, high
|
||||
return round(low, places), round(high, places)
|
||||
|
||||
|
||||
def newcombe(x1: int, n1: int, x2: int, n2: int, z: float = Z95) -> tuple[float, float] | None:
|
||||
"""两比率之差 p1 − p2 的 Newcombe 区间(混合 score 法:两边各自的 Wilson 区间合成)。有一边样本为 0 时没有区间。"""
|
||||
w1, w2 = wilson(x1, n1, z, places=None), wilson(x2, n2, z, places=None)
|
||||
if w1 is None or w2 is None:
|
||||
return None
|
||||
p1, p2 = x1 / n1, x2 / n2
|
||||
(l1, u1), (l2, u2) = w1, w2
|
||||
diff = p1 - p2
|
||||
delta = math.sqrt((p1 - l1) ** 2 + (u2 - p2) ** 2)
|
||||
epsilon = math.sqrt((u1 - p1) ** 2 + (p2 - l2) ** 2)
|
||||
return diff - delta, diff + epsilon
|
||||
|
||||
|
||||
def pooled_z(x1: int, n1: int, x2: int, n2: int) -> tuple[float, float] | None:
|
||||
"""两比率相等的合并比例 z 检验:(z, 双侧 p)。有一边样本为 0 时为 None;合并比例是 0 或 1(两边全是或全不是)时 z 为 0、
|
||||
p 为 1。"""
|
||||
if n1 <= 0 or n2 <= 0:
|
||||
return None
|
||||
pooled = (x1 + x2) / (n1 + n2)
|
||||
se = math.sqrt(pooled * (1 - pooled) * (1 / n1 + 1 / n2))
|
||||
if se == 0:
|
||||
return 0.0, 1.0
|
||||
z = (x1 / n1 - x2 / n2) / se
|
||||
return z, normal_two_sided(z)
|
||||
|
||||
|
||||
def min_expected(x1: int, n1: int, x2: int, n2: int) -> float:
|
||||
"""2×2 表(成功、失败 × 两组)各格的期望频数里最小的那个。"""
|
||||
total = n1 + n2
|
||||
if total <= 0:
|
||||
return 0.0
|
||||
successes, failures = x1 + x2, total - x1 - x2
|
||||
return min(r * c / total for r in (n1, n2) for c in (successes, failures))
|
||||
|
||||
|
||||
def _log_comb(n: int, k: int) -> float:
|
||||
return math.lgamma(n + 1) - math.lgamma(k + 1) - math.lgamma(n - k + 1)
|
||||
|
||||
|
||||
def fisher_exact(a: int, b: int, c: int, d: int) -> float:
|
||||
"""2×2 表 [[a, b], [c, d]] 的 Fisher 精确检验,双侧 p:边际固定时,不比观测到的表更可能出现的那些表的概率之和。"""
|
||||
r1, r2, c1 = a + b, c + d, a + c
|
||||
n = r1 + r2
|
||||
if n == 0:
|
||||
return 1.0
|
||||
lo, hi = max(0, c1 - r2), min(r1, c1)
|
||||
base = _log_comb(n, c1)
|
||||
|
||||
def log_p(x: int) -> float:
|
||||
return _log_comb(r1, x) + _log_comb(r2, c1 - x) - base
|
||||
|
||||
observed = log_p(a) + 1e-7 # 相对余量 1 + 1e-7(对数里约等于加 1e-7),同 R 和 scipy
|
||||
total = sum(math.exp(lp) for lp in (log_p(x) for x in range(lo, hi + 1)) if lp <= observed)
|
||||
return min(1.0, total)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 均值
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Welch(NamedTuple):
|
||||
t: float
|
||||
df: float
|
||||
p: float
|
||||
|
||||
|
||||
def welch(m1: float, v1: float, n1: int, m2: float, v2: float, n2: int) -> Welch | None:
|
||||
"""两组均值相等的 Welch t 检验。v 是样本方差(n − 1 作分母)。有一组样本不到 2 个时为 None(方差算不了)。两组方差都是 0:
|
||||
均值相同 p 为 1,不同 p 为 0。"""
|
||||
if n1 < 2 or n2 < 2:
|
||||
return None
|
||||
s1, s2 = v1 / n1, v2 / n2
|
||||
se2 = s1 + s2
|
||||
if se2 <= 0:
|
||||
return Welch(0.0, math.inf, 1.0) if m1 == m2 else Welch(math.copysign(math.inf, m1 - m2), math.inf, 0.0)
|
||||
t = (m1 - m2) / math.sqrt(se2)
|
||||
denom = (s1 * s1 / (n1 - 1) if s1 else 0.0) + (s2 * s2 / (n2 - 1) if s2 else 0.0)
|
||||
df = se2 * se2 / denom if denom > 0 else math.inf
|
||||
return Welch(t, df, t_two_sided(t, df))
|
||||
|
||||
|
||||
def welch_interval(m1: float, v1: float, n1: int, m2: float, v2: float, n2: int,
|
||||
level: float = 0.95) -> tuple[float, float] | None:
|
||||
"""m1 − m2 的 Welch 区间(缺省 95%)。算不了检验时也没有区间。"""
|
||||
test = welch(m1, v1, n1, m2, v2, n2)
|
||||
if test is None:
|
||||
return None
|
||||
se = math.sqrt(v1 / n1 + v2 / n2)
|
||||
if se == 0:
|
||||
return m1 - m2, m1 - m2
|
||||
q = t_ppf(0.5 + level / 2.0, test.df)
|
||||
return m1 - m2 - q * se, m1 - m2 + q * se
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 多重比较
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def holm(pvalues: Sequence[float | None]) -> list[float | None]:
|
||||
"""Holm 逐步校正:按原始 p 从小到大,第 i 个(从 1 数)乘以 m − i + 1,再取到它为止的最大值,不超过 1。None(这一对算不了)
|
||||
不计入次数、原样留着。"""
|
||||
present = sorted((p, i) for i, p in enumerate(pvalues) if p is not None)
|
||||
m = len(present)
|
||||
out: list[float | None] = [None] * len(pvalues)
|
||||
running = 0.0
|
||||
for rank, (p, i) in enumerate(present):
|
||||
running = max(running, min(1.0, (m - rank) * p))
|
||||
out[i] = running
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 趋势
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _sign(x: Any) -> int:
|
||||
return (x > 0) - (x < 0)
|
||||
|
||||
|
||||
class MannKendall(NamedTuple):
|
||||
s: int
|
||||
var: float # S 的方差(并列修正后)
|
||||
z: float | None # 正态近似的 z(精确分布时为 None)
|
||||
p: float | Fraction # 双侧 p;精确分布时是 Fraction
|
||||
exact: bool
|
||||
|
||||
|
||||
#: 精确分布最多到几期:再多 n! 太大,正态近似已经足够
|
||||
EXACT_MAX_N = 10
|
||||
|
||||
|
||||
def _inversion_counts(n: int) -> list[int]:
|
||||
"""1..n 的排列里,逆序数为 k 的排列有几个(k = 0 … n(n−1)/2)。递推:第 m 个数插进 m 个位置,逆序数各加 0 … m−1。"""
|
||||
counts = [1]
|
||||
for m in range(2, n + 1):
|
||||
nxt = [0] * (len(counts) + m - 1)
|
||||
for k, c in enumerate(counts):
|
||||
for j in range(m):
|
||||
nxt[k + j] += c
|
||||
counts = nxt
|
||||
return counts
|
||||
|
||||
|
||||
def _scaled(values: Sequence[Any]) -> tuple[list[int], int] | None:
|
||||
"""一组精确的数(整数、Fraction)乘上公分母换成整数:(整数, 公分母)。有浮点的为 None。两两相减、比大小在整数上做,
|
||||
比 Fraction 快得多,结果一样。"""
|
||||
if not all(isinstance(v, (int, Fraction)) and not isinstance(v, bool) for v in values):
|
||||
return None
|
||||
scale = math.lcm(*(Fraction(v).denominator for v in values)) if values else 1
|
||||
return [int(Fraction(v) * scale) for v in values], scale
|
||||
|
||||
|
||||
def mann_kendall(values: Sequence[Any]) -> MannKendall:
|
||||
"""Mann–Kendall 趋势检验:S = Σ_{i<j} sgn(y_j − y_i),双侧 p。n ≤ EXACT_MAX_N 且没有并列时用精确分布(没有并列时
|
||||
S = N − 2·逆序数,N = n(n−1)/2,逆序数的分布递推算出);否则正态近似:Var(S) = [n(n−1)(2n+5) − Σt(t−1)(2t+5)] / 18
|
||||
(t 是每组并列的个数),z = (S − sgn S) / √Var(连续性校正)。"""
|
||||
n = len(values)
|
||||
scaled = _scaled(values)
|
||||
ys = scaled[0] if scaled is not None else list(values)
|
||||
s = sum(_sign(ys[j] - ys[i]) for i in range(n) for j in range(i + 1, n))
|
||||
ties = [t for t in Counter(ys).values() if t > 1]
|
||||
var = (n * (n - 1) * (2 * n + 5) - sum(t * (t - 1) * (2 * t + 5) for t in ties)) / 18.0
|
||||
if n < 2:
|
||||
return MannKendall(0, 0.0, None, Fraction(1), True)
|
||||
if not ties and n <= EXACT_MAX_N:
|
||||
counts = _inversion_counts(n)
|
||||
pairs = n * (n - 1) // 2
|
||||
hit = sum(c for k, c in enumerate(counts) if abs(pairs - 2 * k) >= abs(s))
|
||||
return MannKendall(s, var, None, Fraction(hit, math.factorial(n)), True)
|
||||
if var <= 0:
|
||||
return MannKendall(s, var, 0.0, 1.0, False)
|
||||
z = (s - _sign(s)) / math.sqrt(var)
|
||||
return MannKendall(s, var, z, normal_two_sided(z), False)
|
||||
|
||||
|
||||
def theil_sen(xs: Sequence[Any], ys: Sequence[Any]) -> Any:
|
||||
"""Theil–Sen 斜率:所有两点(x 不同)连线斜率的中位数。不到两个点为 None。xs、ys 都是精确的数时结果是 Fraction:斜率
|
||||
在公分母上写成整数之比,按 int / int(正确舍入、单调)的浮点排,只把浮点相等的那几个按精确值比,n = 261 的三万多个
|
||||
斜率也不逐个造 Fraction。"""
|
||||
sx, sy = _scaled(xs), _scaled(ys)
|
||||
if sx is None or sy is None:
|
||||
slopes = [(ys[j] - ys[i]) / (xs[j] - xs[i]) for i in range(len(xs)) for j in range(i + 1, len(xs))
|
||||
if xs[j] != xs[i]]
|
||||
return median(slopes) if slopes else None
|
||||
(X, dx), (Y, dy) = sx, sy
|
||||
pairs = []
|
||||
for i in range(len(X)):
|
||||
for j in range(i + 1, len(X)):
|
||||
den = (X[j] - X[i]) * dy
|
||||
if den:
|
||||
num = (Y[j] - Y[i]) * dx
|
||||
pairs.append((num, den) if den > 0 else (-num, -den))
|
||||
if not pairs:
|
||||
return None
|
||||
keys = [num / den for num, den in pairs]
|
||||
order = sorted(range(len(pairs)), key=keys.__getitem__)
|
||||
|
||||
def kth(k: int) -> Fraction:
|
||||
pivot = keys[order[k]]
|
||||
lo, hi = k, k + 1
|
||||
while lo > 0 and keys[order[lo - 1]] == pivot:
|
||||
lo -= 1
|
||||
while hi < len(order) and keys[order[hi]] == pivot:
|
||||
hi += 1
|
||||
block = sorted(Fraction(*pairs[t]) for t in order[lo:hi])
|
||||
return block[k - lo]
|
||||
|
||||
m = len(pairs)
|
||||
return kth(m // 2) if m % 2 else (kth(m // 2 - 1) + kth(m // 2)) / 2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 位置与离散
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
#: 超过这么多个值时中位数先按浮点排(Theil–Sen 的两两斜率有几万个,逐个按 Fraction 比太慢)
|
||||
_FAST_MEDIAN = 512
|
||||
|
||||
|
||||
def _kth(values: Sequence[Any], k: int) -> Any:
|
||||
"""从小到大第 k 个(从 0 数),精确:Fraction 换成浮点是正确舍入的、单调的——浮点小的精确值一定小,排错的只可能是浮点
|
||||
相等的那几个,所以先按浮点排,再只把和第 k 个浮点相等的那一段按精确值排。"""
|
||||
keyed = sorted((float(v), i) for i, v in enumerate(values))
|
||||
pivot = keyed[k][0]
|
||||
lo, hi = k, k + 1
|
||||
while lo > 0 and keyed[lo - 1][0] == pivot:
|
||||
lo -= 1
|
||||
while hi < len(keyed) and keyed[hi][0] == pivot:
|
||||
hi += 1
|
||||
block = sorted(values[i] for _, i in keyed[lo:hi])
|
||||
return block[k - lo]
|
||||
|
||||
|
||||
def median(values: Sequence[Any]) -> Any:
|
||||
"""中位数(偶数个时取中间两个的平均)。Fraction 进 Fraction 出。没有值为 None。"""
|
||||
n = len(values)
|
||||
if n == 0:
|
||||
return None
|
||||
mid = n // 2
|
||||
if n > _FAST_MEDIAN:
|
||||
upper = _kth(values, mid)
|
||||
return upper if n % 2 else (_kth(values, mid - 1) + upper) / 2
|
||||
data = sorted(values)
|
||||
if n % 2:
|
||||
return data[mid]
|
||||
return (data[mid - 1] + data[mid]) / 2
|
||||
|
||||
|
||||
def mad(values: Sequence[Any], center: Any | None = None) -> Any:
|
||||
"""中位绝对偏差:各值离中位数(或给定的中心)的绝对距离的中位数。"""
|
||||
if not values:
|
||||
return None
|
||||
c = median(values) if center is None else center
|
||||
return median([abs(v - c) for v in values])
|
||||
|
||||
|
||||
def rank_desc(x: Any, others: Sequence[Any]) -> int:
|
||||
"""x 在「others + x」里从大到小的名次(并列取最好的名次:比它大的有几个再加 1)。"""
|
||||
return 1 + sum(1 for o in others if o > x)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EXACT_MAX_N", "MannKendall", "Welch", "Z95", "betainc", "fisher_exact", "holm", "mad", "mann_kendall", "median",
|
||||
"min_expected", "newcombe", "normal_cdf", "normal_ppf", "normal_two_sided", "pooled_z", "rank_desc", "t_cdf",
|
||||
"t_ppf", "t_two_sided", "theil_sen", "welch", "welch_interval", "wilson",
|
||||
]
|
||||
@@ -447,9 +447,14 @@ async def _fallback(flt: _Filter, facts: Facts) -> _Range | None:
|
||||
found = await facts(flt.table_name)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
return _facts_range(found, flt.column_name)
|
||||
|
||||
|
||||
def _facts_range(found: Any, column: str) -> _Range | None:
|
||||
"""剖析事实(last_facts 的一项)里这一列的范围:先剖析的 date_ranges,再取值观察的最新日期。"""
|
||||
if not isinstance(found, Mapping):
|
||||
return None
|
||||
wanted = name_key(flt.column_name)
|
||||
wanted = name_key(column)
|
||||
for item in found.get("date_ranges") or []:
|
||||
if isinstance(item, Mapping) and name_key(str(item.get("column") or "")) == wanted:
|
||||
lo, hi = _as_day(item.get("min")), _as_day(item.get("max"))
|
||||
@@ -806,5 +811,88 @@ def no_data_of(snapshot: Any, column: Any) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
|
||||
__all__ = ["BUDGET_S", "Cache", "NO_DATA_SHORT", "PROBE_TIMEOUT_S", "Window", "filters", "narrowed", "no_data_of",
|
||||
"result_gaps", "sql_gaps", "zero_columns"]
|
||||
# ==========================================================================
|
||||
# 给分析的可比性检查用的几个小函数(app/analysis/comparable.py、probe.py:窗口长度、整期)。读法、写法、探查都和上面同一套,
|
||||
# 只是把中间结果交出去:分析结论要把现查的范围、在库里算出的界记下来,复算时用记下的值
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def parse_sql(sql: Any, read: str | None) -> Any:
|
||||
"""一条查询解析成语法树(同覆盖检查:太长、带模板占位、不止一条、不是查询的为 None)。"""
|
||||
return _parse(sql, read)
|
||||
|
||||
|
||||
def as_day(value: Any, kind: str | None = None) -> _Day | None:
|
||||
"""一个值归一成日期(DATE / DATETIME、'YYYY-MM-DD'、'YYYYMMDD'、8 位整数);认不出为 None。"""
|
||||
return _as_day(value, kind)
|
||||
|
||||
|
||||
def literal_bound(node: Any) -> tuple[bool, _Day | None]:
|
||||
"""时间条件的一个界:(是不是字面量, 字面量归一成的日期)。不是字面量的是按运行当天现算的,要在库里算。"""
|
||||
return (True, _literal_day(node)) if _is_literal(node) else (False, None)
|
||||
|
||||
|
||||
def window_of(conds: list[tuple[str, Any, bool]], days: Mapping[int, _Day | None]) -> Window | None:
|
||||
"""几个条件合成一段区间(同 _window:下界取最晚、上界取最早,「>」「<」的界不带时刻时按天挪一天)。days 是
|
||||
{id(界的节点): 日期}。不核对列和界的写法比不比得起来:分析结论里表的范围只记了日期。"""
|
||||
return _window(conds, days, _Range(None, None, "none"), None)
|
||||
|
||||
|
||||
def _range_out(data: _Range | None) -> dict[str, Any] | None:
|
||||
if data is None:
|
||||
return None
|
||||
return {"min": data.low.day.isoformat() if data.low else None,
|
||||
"max": data.high.day.isoformat() if data.high else None,
|
||||
"basis": data.basis, "at": data.day, "empty": data.empty}
|
||||
|
||||
|
||||
def facts_range(found: Any, column: str) -> dict[str, Any] | None:
|
||||
"""剖析事实(catalog_profile_jobs.last_facts 的一项)里这一列的范围,形状同 column_range;没有为 None。"""
|
||||
return _range_out(_facts_range(found, column))
|
||||
|
||||
|
||||
async def column_range(source: Any, table: str, column: str, *, facts: Facts | None = None,
|
||||
cache: Cache | None = None) -> dict[str, Any] | None:
|
||||
"""这一列在这张表里的实际范围:现查 MIN / MAX(只读守卫、PROBE_TIMEOUT_S 秒、同一张表 LIVE_TTL_S 秒内不重查),查不成
|
||||
退回剖析、取值观察(facts)。{min, max, basis, at, empty}:min / max 是 ISO 日期(不知道的那一头为 None),basis 是 live /
|
||||
profile / observed,at 是剖析、观察的日期(现查为 None),表里这一列一个值都没有时 empty 为真。都没有为 None。从不抛异常。"""
|
||||
from sqlglot import exp
|
||||
|
||||
try:
|
||||
parts = [p for p in str(table).split(".") if p]
|
||||
node = exp.Table(this=exp.to_identifier(parts[-1]),
|
||||
**({"db": exp.to_identifier(parts[-2])} if len(parts) > 1 else {}))
|
||||
flt = _Filter((), node, str(table), exp.column(str(column)), str(column))
|
||||
return _range_out(await _range(source, flt, _read(getattr(source, "kind", None)), cache or Cache(), facts))
|
||||
except Exception: # noqa: BLE001 - 探查失败就当不知道
|
||||
logger.info("可比性检查的范围探查没有成功(数据源 %s)", getattr(source, "name", ""), exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
async def bound_values(source: Any, texts: list[str]) -> list[Any] | None:
|
||||
"""按运行当天现算的几个界(SQL 表达式,按这个数据源的方言写)在库里算出来:一条 SELECT 界1, 界2 …(Oracle 加 FROM
|
||||
DUAL),走只读守卫和 PROBE_TIMEOUT_S。交回各个值(日期、时刻按 ISO 写,其余照库里交回的);查不成为 None。从不抛异常。"""
|
||||
if not texts:
|
||||
return []
|
||||
read = _read(getattr(source, "kind", None))
|
||||
try:
|
||||
found = await _probe(source, _select_sql([f"{t} AS b{i}" for i, t in enumerate(texts)], read))
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
if found is None:
|
||||
return None
|
||||
values = list(found[0]) + [None] * len(texts)
|
||||
out: list[Any] = []
|
||||
for value in values[:len(texts)]:
|
||||
if isinstance(value, datetime):
|
||||
out.append(value.isoformat(sep=" ", timespec="seconds"))
|
||||
elif isinstance(value, date):
|
||||
out.append(value.isoformat())
|
||||
else:
|
||||
out.append(value)
|
||||
return out
|
||||
|
||||
|
||||
__all__ = ["BUDGET_S", "Cache", "NO_DATA_SHORT", "PROBE_TIMEOUT_S", "Window", "as_day", "bound_values", "column_range",
|
||||
"facts_range", "filters", "literal_bound", "narrowed", "no_data_of", "parse_sql", "result_gaps", "sql_gaps",
|
||||
"window_of", "zero_columns"]
|
||||
|
||||
@@ -22,25 +22,15 @@ from collections import Counter
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
#: 95% 区间的 z
|
||||
Z95 = 1.959963984540054
|
||||
#: Wilson 区间和 95% 区间的 z 在分析的统计里(总方案 D13:分析和评测共用一套纯 Python 的统计),这里引用、行为不变
|
||||
from app.analysis.stats import Z95, wilson
|
||||
|
||||
#: 问数据页等复核最多这么久(毫秒,同 answer.REVIEW_PAGE_WAIT_S)
|
||||
REVIEW_PAGE_WAIT_MS = 25_000
|
||||
TIERS = ("formal", "degraded", "withheld", "explore")
|
||||
TIMING_KEYS = ("build_ms", "run_wall_ms", "run_active_ms", "review_ms", "total_ms")
|
||||
|
||||
|
||||
def wilson(passed: int, n: int, z: float = Z95) -> tuple[float, float] | None:
|
||||
"""Wilson 区间(缺省 95%)。n 为 0 时没有区间。"""
|
||||
if n <= 0:
|
||||
return None
|
||||
p = passed / n
|
||||
denom = 1 + z * z / n
|
||||
centre = (p + z * z / (2 * n)) / denom
|
||||
half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom
|
||||
return round(max(0.0, centre - half), 4), round(min(1.0, centre + half), 4)
|
||||
|
||||
|
||||
def mcnemar_exact(only_a: int, only_b: int) -> float:
|
||||
"""精确 McNemar 双侧 p 值:不一致的那几题里,「只 A 对」服从 Binomial(b + c, 0.5)。没有不一致的题时为 1。"""
|
||||
n = only_a + only_b
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""分析算子测试用的样例:虚构零售库 shop(地区华东、华南、华北、西南,渠道线上、线下,销售额、订单数、退款),表名、列名、
|
||||
数字都是编的。analyze 跑一次 analysis.run,顺手用 recompute 复算一遍、核对逐字段相同(contract.same),所以每个用例都同时
|
||||
钉住「复算和运行逐字段相同」。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from app import analysis
|
||||
from app.analysis.contract import CallError, effective_settings, same
|
||||
from app.core.artifact_store import canonical_json, content_hash
|
||||
|
||||
#: 测试里的「运行当天」
|
||||
TODAY = date(2026, 10, 21)
|
||||
|
||||
|
||||
def snap(ref: str, sql: str, columns: Sequence[str], rows: Sequence[Sequence[Any]], *,
|
||||
types: Mapping[str, str] | None = None, source: str = "shop", truncated: bool = False,
|
||||
masked: Sequence[str] = (), **extra: Any) -> dict[str, Any]:
|
||||
"""一份输入快照(InputSnapshot 的形状)。artifact 按内容算,复算按它取回。"""
|
||||
body = {"source": source, "sql": sql, "columns": list(columns), "rows": [list(r) for r in rows],
|
||||
"truncated": truncated, **({"column_types": dict(types)} if types else {}), **extra}
|
||||
return {"ref": ref, "artifact": content_hash(canonical_json(body)), "node_id": "q1", "masked": list(masked), **body}
|
||||
|
||||
|
||||
def item(value: Any, status: str = "confirmed", source: str = "human") -> dict[str, Any]:
|
||||
return {"value": value, "source": source, "status": status}
|
||||
|
||||
|
||||
def notes(*, columns: Mapping[str, Mapping[str, Any]] | None = None, business_date: str | None = None) -> dict[str, Any]:
|
||||
"""一张表的目录 notes:columns 是 {列: {label / measure: 值或 item(...)}}。"""
|
||||
out: dict[str, Any] = {}
|
||||
if columns:
|
||||
out["columns"] = {c: {k: v if isinstance(v, Mapping) else item(v) for k, v in spec.items()}
|
||||
for c, spec in columns.items()}
|
||||
if business_date:
|
||||
out["business_date"] = item({"column": business_date, "rule": "下单日", "timezone": "Asia/Shanghai"})
|
||||
return out
|
||||
|
||||
|
||||
#: orders 表的目录:地区、渠道、实收金额(流量)、退款单数
|
||||
ORDERS = notes(columns={"region": {"label": "地区"}, "channel": {"label": "渠道"},
|
||||
"amount": {"label": "销售额", "measure": "flow"},
|
||||
"refund_flag": {"label": "退款单数", "measure": "flow"},
|
||||
"order_date": {"label": "下单日"}},
|
||||
business_date="order_date")
|
||||
CATALOGS = {"shop": {"orders": ORDERS}}
|
||||
|
||||
|
||||
async def analyze(op: str, mode: str | None = None, *, snaps: Sequence[Mapping[str, Any]], today: date = TODAY,
|
||||
catalogs: Mapping[str, Any] | None = None, probe: Any = None, facts: Any = None,
|
||||
parents: Mapping[str, Mapping[str, Any]] | None = None, settings: Mapping[str, Any] | None = None,
|
||||
hypothesis: str | None = None, check: bool = True, **params: Any) -> dict[str, Any]:
|
||||
"""跑一次 analysis.run;check 时再复算一遍,核对和封存的逐字段相同(hypothesis 除外)。"""
|
||||
by_ref = {s["ref"]: s for s in snaps}
|
||||
request: dict[str, Any] = {"op": op, "mode": mode, "params": params}
|
||||
if hypothesis is not None:
|
||||
request["hypothesis"] = hypothesis
|
||||
context: dict[str, Any] = {"today": today, "catalogs": CATALOGS if catalogs is None else catalogs}
|
||||
if probe is not None:
|
||||
context["probe"] = probe
|
||||
if facts is not None:
|
||||
context["facts"] = facts
|
||||
result = await analysis.run(request, by_ref.get, settings=effective_settings(settings), context=context,
|
||||
parents=(parents or {}).get)
|
||||
if check:
|
||||
again = recomputed(result, snaps, parents)
|
||||
assert same(result, again), _diff(result, again)
|
||||
assert again["hypothesis"] == result["hypothesis"]
|
||||
return dict(result)
|
||||
|
||||
|
||||
def recomputed(result: Mapping[str, Any], snaps: Sequence[Mapping[str, Any]],
|
||||
parents: Mapping[str, Mapping[str, Any]] | None = None) -> dict[str, Any]:
|
||||
"""按结论里记下的复算:快照按 artifact 取回(只交快照内容本身,不交入口加的出处),上一层结论按内容哈希取回。"""
|
||||
by_artifact = {s["artifact"]: {k: v for k, v in s.items() if k not in ("ref", "artifact", "node_id", "masked")}
|
||||
for s in snaps}
|
||||
by_hash = {content_hash(canonical_json(r)): r for r in (parents or {}).values()}
|
||||
return dict(analysis.recompute(result, by_artifact.get, load_result=by_hash.get)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _diff(a: Mapping[str, Any], b: Mapping[str, Any]) -> str:
|
||||
keys = sorted(k for k in set(a) | set(b) if a.get(k) != b.get(k))
|
||||
return ";".join(f"{k}: {json.dumps(a.get(k), ensure_ascii=False)} ≠ {json.dumps(b.get(k), ensure_ascii=False)}"
|
||||
for k in keys)
|
||||
|
||||
|
||||
async def call_error(op: str, mode: str | None = None, **kwargs: Any) -> CallError:
|
||||
"""跑一次,期望是调用错误。"""
|
||||
try:
|
||||
await analyze(op, mode, check=False, **kwargs)
|
||||
except CallError as e:
|
||||
return e
|
||||
raise AssertionError("应当是调用错误")
|
||||
|
||||
|
||||
def fixed_prober(ranges: Mapping[tuple[str, str], tuple[str | None, str | None]] | None = None, *,
|
||||
bounds: Mapping[str, str] | None = None, at: str = "2026-10-21T09:00:00") -> Any:
|
||||
"""测试用的探查器:ranges 是 {(表, 列): (最早, 最晚)},bounds 是 {界的 SQL: 日期};没写的查不到。"""
|
||||
|
||||
class Prober:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, ...]] = []
|
||||
|
||||
async def __call__(self, source: str, table: str, column: str) -> dict[str, Any] | None:
|
||||
self.calls.append((source, table, column))
|
||||
found = (ranges or {}).get((table, column))
|
||||
if found is None:
|
||||
return None
|
||||
return {"source": source, "table": table, "column": column, "min": found[0], "max": found[1],
|
||||
"basis": "live", "at": at}
|
||||
|
||||
async def bounds(self, source: str, expr: str) -> dict[str, Any] | None:
|
||||
self.calls.append((source, expr))
|
||||
value = (bounds or {}).get(expr)
|
||||
return None if value is None else {"source": source, "expr": expr, "value": value, "basis": "live", "at": at}
|
||||
|
||||
def dialect(self, source: str) -> str | None:
|
||||
return "sqlite"
|
||||
|
||||
return Prober()
|
||||
|
||||
|
||||
__all__ = ["CATALOGS", "ORDERS", "TODAY", "analyze", "call_error", "fixed_prober", "item", "notes", "recomputed",
|
||||
"snap"]
|
||||
@@ -0,0 +1,182 @@
|
||||
"""可比性检查(app/analysis/comparable.py,细案 01 §6):只差日期 / 差了别的筛选 / 结构不同、窗口不等(流量不可比、比率只提醒)、
|
||||
本期不完整、周末天数不同、探查失败退回剖析事实、都没有就核对不了。每个用例都顺带核对复算和运行逐字段相同(复算用记下的
|
||||
probes,不现查)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.analysis import comparable_reasons
|
||||
from tests.analysis_samples import analyze, fixed_prober, snap
|
||||
|
||||
WINDOW = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= '{a}' AND order_date < '{b}'"
|
||||
RANGE = {("orders", "order_date"): ("2025-01-01", "2026-10-20")}
|
||||
|
||||
|
||||
def period(ref, a, b, sql=WINDOW, **kw):
|
||||
return snap(ref, sql.format(a=a, b=b), ["gmv"], [[100]], types={"gmv": "number"}, **kw)
|
||||
|
||||
|
||||
async def compare(base, target, *, probe=None, **params):
|
||||
params = {"base_query": base["ref"], "target_query": target["ref"], **params}
|
||||
return await analyze("comparable", snaps=[base, target], probe=probe, **params)
|
||||
|
||||
|
||||
def checks(result):
|
||||
return {r[0]: r[1] for r in result["table"]["rows"]}
|
||||
|
||||
|
||||
async def test_two_weeks_that_differ_only_in_dates_are_comparable():
|
||||
result = await compare(period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19"),
|
||||
probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["method"] == "comparable@1" and result["mode"] is None
|
||||
assert result["verdict"] == {"code": "comparable", "subjects": [], "direction": None}
|
||||
assert result["phrase"] == "两期可比"
|
||||
assert checks(result) == {"同口径": "通过", "窗口长度": "通过", "整期": "通过", "星期构成": "通过", "数据问题": "通过"}
|
||||
assert result["values"]["base_days"] == "7" and result["values"]["target_start"] == "2026-10-12"
|
||||
assert result["values"]["window"] == "pass" and result["settings"] == {}
|
||||
assert result["params"]["today"] == "2026-10-21" and result["params"]["measure"] == "gmv"
|
||||
assert [p["column"] for p in result["probes"]] == ["order_date"]
|
||||
assert result["table"]["columns"] == ["check", "status", "base", "target", "note"]
|
||||
|
||||
|
||||
async def test_other_literals_differing_mean_two_segments_not_two_periods():
|
||||
sql = WINDOW + " AND region = '{r}'"
|
||||
base = snap("Q1", sql.format(a="2026-10-05", b="2026-10-12", r="华南"), ["gmv"], [[100]])
|
||||
target = snap("Q2", sql.format(a="2026-10-12", b="2026-10-19", r="华东"), ["gmv"], [[100]])
|
||||
result = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["verdict"]["code"] == "incomparable"
|
||||
assert result["phrase"] == "两期不可比:两边除了日期,筛选的还不一样"
|
||||
assert comparable_reasons(result) == ["两边除了日期,筛选的还不一样"]
|
||||
|
||||
|
||||
async def test_a_different_structure_says_where_it_differs():
|
||||
base = period("Q1", "2026-10-05", "2026-10-12")
|
||||
target = period("Q2", "2026-10-12", "2026-10-19", sql=WINDOW + " AND status = 'paid'")
|
||||
result = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["verdict"]["code"] == "incomparable"
|
||||
assert result["phrase"] == "两期不可比:两份查询的口径结构不同(差在筛选条件)"
|
||||
other = await compare(base, period("Q2", "2026-10-12", "2026-10-19", source="crm"), measure="gmv")
|
||||
assert "两份查询来自不同的数据源" in other["phrase"]
|
||||
|
||||
|
||||
async def test_unequal_windows_fail_for_flows_and_only_warn_otherwise():
|
||||
base, target = period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-15")
|
||||
flow = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv") # SUM(amount):目录标的流量
|
||||
assert flow["verdict"]["code"] == "incomparable" and checks(flow)["窗口长度"] == "不通过"
|
||||
assert "两段的天数不同:7 天对 3 天" in flow["phrase"]
|
||||
avg_sql = "SELECT AVG(amount) AS aov FROM orders WHERE order_date >= '{a}' AND order_date < '{b}'"
|
||||
ratio = await compare(period("Q1", "2026-10-05", "2026-10-12", sql=avg_sql),
|
||||
period("Q2", "2026-10-12", "2026-10-15", sql=avg_sql), probe=fixed_prober(RANGE),
|
||||
measure="gmv")
|
||||
assert ratio["verdict"]["code"] == "conditional" and checks(ratio)["窗口长度"] == "提醒"
|
||||
unlabeled = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv", catalogs={})
|
||||
assert unlabeled["verdict"]["code"] == "conditional"
|
||||
assert any(c["code"] == "measure_unlabeled" for c in unlabeled["caveats"])
|
||||
|
||||
|
||||
async def test_whole_months_count_as_the_same_length():
|
||||
result = await compare(period("Q1", "2026-08-01", "2026-09-01"), period("Q2", "2026-09-01", "2026-10-01"),
|
||||
probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["verdict"]["code"] == "comparable" and result["values"]["base_days"] == "31"
|
||||
|
||||
|
||||
async def test_an_unfinished_period_is_not_a_whole_period():
|
||||
result = await compare(period("Q1", "2026-10-12", "2026-10-19"), period("Q2", "2026-10-19", "2026-10-26"),
|
||||
probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["verdict"]["code"] == "incomparable"
|
||||
assert result["phrase"] == "两期不可比:本期数据只到 2026-10-20,还差 5 天"
|
||||
late = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-16")})
|
||||
behind = await compare(period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19"),
|
||||
probe=late, measure="gmv")
|
||||
assert behind["phrase"] == "两期不可比:本期数据只到 2026-10-16,还差 2 天"
|
||||
|
||||
|
||||
async def test_weekend_composition_of_short_windows():
|
||||
# 周一到周三 对 周五到周日
|
||||
result = await compare(period("Q1", "2026-10-05", "2026-10-08"), period("Q2", "2026-10-16", "2026-10-19"),
|
||||
probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["verdict"]["code"] == "conditional"
|
||||
assert result["phrase"] == "两期有条件可比:两段的周末天数不同:0 天对 2 天"
|
||||
|
||||
|
||||
async def test_a_failed_probe_falls_back_to_profile_facts_then_says_it_cannot_check():
|
||||
facts = {"shop": {"orders": {"at": "2026-10-20T03:00:00", "date_ranges": [
|
||||
{"column": "order_date", "min": "2025-01-01", "max": "2026-10-19"}]}}}
|
||||
base, target = period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19")
|
||||
fallback = await compare(base, target, probe=fixed_prober({}), facts=facts, measure="gmv")
|
||||
assert fallback["verdict"]["code"] == "comparable" and fallback["phrase"] == "两期可比"
|
||||
assert fallback["probes"] == [{"source": "shop", "table": "orders", "column": "order_date", "min": "2025-01-01",
|
||||
"max": "2026-10-19", "basis": "profile", "at": "2026-10-20"}]
|
||||
nothing = await compare(base, target, measure="gmv")
|
||||
assert nothing["verdict"]["code"] == "comparable"
|
||||
assert nothing["phrase"] == "两期可比,整期核对不了" and nothing["probes"] == []
|
||||
|
||||
|
||||
async def test_bounds_computed_from_the_run_day_are_evaluated_in_the_database_and_recorded():
|
||||
sql = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', '{a}') AND order_date < date('now', '{b}')"
|
||||
base = snap("Q1", sql.format(a="-16 days", b="-9 days"), ["gmv"], [[100]])
|
||||
target = snap("Q2", sql.format(a="-9 days", b="-2 days"), ["gmv"], [[90]])
|
||||
bounds = {"DATE('now', '-16 days')": "2026-10-05", "DATE('now', '-9 days')": "2026-10-12",
|
||||
"DATE('now', '-2 days')": "2026-10-19"}
|
||||
probe = fixed_prober(RANGE, bounds=bounds)
|
||||
result = await compare(base, target, probe=probe, measure="gmv")
|
||||
assert result["verdict"]["code"] == "comparable"
|
||||
assert checks(result)["同口径"] == "通过" # 只差按运行当天现算的界
|
||||
assert result["values"]["base_start"] == "2026-10-05" and result["values"]["target_end"] == "2026-10-18"
|
||||
exprs = [p["expr"] for p in result["probes"] if "expr" in p]
|
||||
assert exprs == ["DATE('now', '-16 days')", "DATE('now', '-9 days')", "DATE('now', '-2 days')"]
|
||||
assert all(p["basis"] == "live" for p in result["probes"])
|
||||
# 界算不出来:窗口核对不了,不判不可比
|
||||
blind = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert blind["verdict"]["code"] == "comparable" and "窗口长度" in blind["phrase"]
|
||||
|
||||
|
||||
async def test_data_problems_on_the_snapshots():
|
||||
base = period("Q1", "2026-10-05", "2026-10-12")
|
||||
empty = period("Q2", "2026-10-12", "2026-10-19", no_data=[{"columns": ["gmv"], "table": "orders",
|
||||
"column": "order_date", "message": "没有数据"}])
|
||||
result = await compare(base, empty, probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["verdict"]["code"] == "incomparable" and "本期查询的区间里这张表没有数据" in result["phrase"]
|
||||
checked = period("Q2", "2026-10-12", "2026-10-19", checks=[{"code": "fanout_sum", "level": "error",
|
||||
"message": "关联订单明细后对订单金额求和,会重复计算"}])
|
||||
warned = await compare(base, checked, probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert warned["verdict"]["code"] == "conditional"
|
||||
assert "本期查询的 SQL 检查有错误:关联订单明细后对订单金额求和,会重复计算" in warned["phrase"]
|
||||
|
||||
|
||||
async def test_one_snapshot_grouped_by_period_is_naturally_the_same_query():
|
||||
sql = "SELECT order_date AS day, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-01' GROUP BY 1"
|
||||
rows = [[f"2026-10-{d:02d}", 100] for d in range(1, 20)]
|
||||
s = snap("Q1", sql, ["day", "gmv"], rows, types={"day": "text", "gmv": "number"})
|
||||
probe = fixed_prober(RANGE)
|
||||
same_weekday = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="day", base="2026-10-05",
|
||||
target="2026-10-12")
|
||||
assert same_weekday["verdict"]["code"] == "comparable" and same_weekday["params"]["measure"] == "gmv"
|
||||
assert same_weekday["params"]["grain"] == "day"
|
||||
weekend = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="day", base="2026-10-09",
|
||||
target="2026-10-10")
|
||||
assert weekend["phrase"] == "两期有条件可比:两段的周末天数不同:0 天对 1 天"
|
||||
# 期次有一截落在 SQL 的时间条件之外:基期被截掉了;本期(10 月)还没过完
|
||||
monthly_rows = [["2026-09-01", 100], ["2026-10-01", 90]]
|
||||
month = snap("Q1", "SELECT DATE(order_date, 'start of month') AS month, SUM(amount) AS gmv FROM orders "
|
||||
"WHERE order_date >= '2026-09-05' GROUP BY 1", ["month", "gmv"], monthly_rows)
|
||||
partial = await analyze("comparable", snaps=[month], probe=probe, query="Q1", period="month", base="2026-09-01",
|
||||
target="2026-10-01", measure="gmv")
|
||||
assert partial["verdict"]["code"] == "incomparable"
|
||||
assert "基期数据从 2026-09-05 才开始,缺前 4 天" in partial["phrase"]
|
||||
assert "本期数据只到 2026-10-20,还差 11 天" in partial["phrase"]
|
||||
|
||||
|
||||
async def test_reasons_are_split_one_sentence_each():
|
||||
result = await compare(period("Q1", "2026-10-05", "2026-10-08"), period("Q2", "2026-10-16", "2026-10-21"),
|
||||
probe=fixed_prober(RANGE), measure="gmv")
|
||||
reasons = comparable_reasons(result)
|
||||
assert "两段的天数不同:3 天对 5 天" in reasons and "本期数据只到 2026-10-20,还差 0 天" not in reasons
|
||||
assert all(";" not in r for r in reasons)
|
||||
assert comparable_reasons({"table": None}) == []
|
||||
|
||||
|
||||
async def test_a_masked_date_column_is_neither_used_nor_probed():
|
||||
probe = fixed_prober(RANGE)
|
||||
result = await compare(period("Q1", "2026-10-05", "2026-10-12", masked=["order_date"]),
|
||||
period("Q2", "2026-10-12", "2026-10-19", masked=["order_date"]), probe=probe, measure="gmv")
|
||||
assert probe.calls == [] and result["probes"] == []
|
||||
assert result["verdict"]["code"] == "comparable" and "窗口长度" in result["phrase"]
|
||||
@@ -0,0 +1,420 @@
|
||||
"""变化拆解(app/analysis/decompose.py,细案 01 §2):三种恒等式逐位成立、辛普森、抵消、不闭合、下钻闭合与超深、新出现 /
|
||||
消失的段、比率当可加的调用错误……每个用例都顺带核对复算和运行逐字段相同(tests/analysis_samples.analyze)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import random
|
||||
from decimal import Decimal
|
||||
from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
|
||||
from app.analysis.decompose import lmdi, ratio_effects
|
||||
from tests.analysis_samples import analyze, call_error, fixed_prober, notes, snap
|
||||
|
||||
SQL = "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week"
|
||||
TYPES = {"region": "text", "week": "text", "gmv": "number"}
|
||||
WEEKLY = [["华东", "2026-W41", 820], ["华东", "2026-W42", 700], ["华北", "2026-W41", 600], ["华北", "2026-W42", 560],
|
||||
["华南", "2026-W41", 700], ["华南", "2026-W42", 666], ["西南", "2026-W41", 100], ["西南", "2026-W42", 100]]
|
||||
|
||||
|
||||
def weekly(rows=WEEKLY, **kw):
|
||||
return snap("Q2", SQL, ["region", "week", "gmv"], rows, types=TYPES, **kw)
|
||||
|
||||
|
||||
async def additive(rows=WEEKLY, *, snaps=None, **params):
|
||||
params = {"query": "Q2", "by": ["region"], "period": "week", "base": "2026-W41", "target": "2026-W42",
|
||||
"measure": "gmv", **params}
|
||||
return await analyze("decompose", "additive", snaps=snaps or [weekly(rows)], **params)
|
||||
|
||||
|
||||
def _rows(base: dict, target: dict) -> list[list]:
|
||||
return [[k, "2026-W41", v] for k, v in base.items()] + [[k, "2026-W42", v] for k, v in target.items()]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# additive
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_driver_is_named_with_its_share_and_the_whole_shape_is_filled():
|
||||
result = await additive(label="销售额", hypothesis="华东拖累")
|
||||
assert result["method"] == "decompose.additive@1" and result["status"] == "ok"
|
||||
assert result["verdict"] == {"code": "driver", "subjects": [["华东"]], "direction": "down"}
|
||||
assert result["phrase"] == "销售额下降 194(-8.7%),主要来自华东(-120,占 61.9%)"
|
||||
assert result["values"]["delta_total"] == "-194" and result["values"]["top_key"] == "华东"
|
||||
assert result["values"]["top_share"] == "0.618556701031" and result["values"]["segments"] == "4"
|
||||
table = result["table"]
|
||||
assert table["columns"] == ["region", "base", "target", "delta", "pct", "share", "flag"]
|
||||
assert table["labels"]["region"] == "地区" and table["labels"]["share"] == "占变化的比重"
|
||||
assert [r[0] for r in table["rows"]] == ["华东", "华北", "华南", "西南"] # 按变化的绝对值从大到小
|
||||
assert result["settings"] == {"main_share": 0.5, "concentration": 0.8, "offset_ratio": 0.25, "flat_pct": 0.005,
|
||||
"table_top_k": 10}
|
||||
assert result["inputs"] == [{"ref": "Q2", "artifact": weekly()["artifact"], "role": "data",
|
||||
"columns": ["region", "week", "gmv"], "rows": "all"}]
|
||||
assert result["hypothesis"] == "华东拖累" and result["probes"] == []
|
||||
assert result["criterion"].endswith("以各段之和为总数")
|
||||
assert {c["code"] for c in result["caveats"]} == {"headline_basis"}
|
||||
assert result["params"]["catalog"] == {"labels": {"region": "地区"}, "measures": {"gmv": "flow"}}
|
||||
|
||||
|
||||
async def test_the_segments_add_up_to_the_total_digit_for_digit():
|
||||
rng = random.Random(7)
|
||||
regions = ["华东", "华南", "华北", "西南", "东北", "西北"]
|
||||
for _ in range(20):
|
||||
base = {r: round(rng.uniform(0, 5000), rng.choice([0, 2, 3])) for r in regions}
|
||||
target = {r: round(rng.uniform(0, 5000), rng.choice([0, 2, 3])) for r in regions}
|
||||
result = await additive(_rows(base, target))
|
||||
rows = result["table"]["rows"]
|
||||
assert sum(Decimal(r[3]) for r in rows) == Decimal(result["values"]["delta_total"])
|
||||
assert sum(Decimal(r[1]) for r in rows) == Decimal(result["values"]["base_total"])
|
||||
assert sum(Decimal(r[2]) for r in rows) == Decimal(result["values"]["target_total"])
|
||||
assert Decimal(result["values"]["target_total"]) - Decimal(result["values"]["base_total"]) == \
|
||||
Decimal(result["values"]["delta_total"])
|
||||
assert result["values"]["gross_up"] and result["values"]["gross_down"]
|
||||
|
||||
|
||||
async def test_concentrated_and_dispersed():
|
||||
base = {f"店{i}": 1000 for i in range(10)}
|
||||
target = {**base, "店0": 1450, "店1": 1400, "店2": 1050, "店3": 1050, "店4": 1050}
|
||||
result = await additive(_rows(base, target), by="region")
|
||||
assert result["verdict"]["code"] == "concentrated"
|
||||
assert result["verdict"]["subjects"] == [["店0"], ["店1"]]
|
||||
assert result["values"]["top_k"] == "2" and result["values"]["top_k_share"] == "0.85"
|
||||
assert "集中在店0、店1(合计占 85.0%)" in result["phrase"]
|
||||
spread = await additive(_rows({r: 1000 for r in "甲乙丙丁戊"}, {r: 1100 for r in "甲乙丙丁戊"}))
|
||||
assert spread["verdict"] == {"code": "dispersed", "subjects": [], "direction": "up"}
|
||||
assert "分散在各地区,没有哪一段占到一半" in spread["phrase"]
|
||||
|
||||
|
||||
async def test_offsetting_segments_are_said_and_a_share_can_pass_one_hundred_percent():
|
||||
result = await additive(_rows({"华东": 1000, "华北": 500, "华南": 500}, {"华东": 850, "华北": 540, "华南": 490}))
|
||||
assert result["values"]["delta_total"] == "-120"
|
||||
assert result["verdict"]["code"] == "driver" and result["values"]["top_share"] == "1.25"
|
||||
offset = next(c for c in result["caveats"] if c["code"] == "offset")
|
||||
assert offset["text"] == "下降的段合计 -160,上升的段合计 +40,互相抵消了一部分"
|
||||
assert result["phrase"] == "销售额下降 120(-6.0%),主要来自华东(-150,占 125.0%);下降的段合计 -160,上升的段合计 +40,互相抵消了一部分"
|
||||
|
||||
|
||||
async def test_flat_with_offsets_says_what_moved_inside():
|
||||
result = await additive(_rows({"华东": 1000, "华北": 1000, "华南": 220}, {"华东": 1120, "华北": 882, "华南": 220}))
|
||||
assert result["verdict"] == {"code": "flat", "subjects": [], "direction": None}
|
||||
assert result["phrase"] == "销售额基本持平(+0.1%),内部此消彼长:华东 +120、华北 -118"
|
||||
|
||||
|
||||
async def test_a_headline_that_does_not_close_stops_the_concentration_verdict():
|
||||
head = snap("Q1", "SELECT week, SUM(amount) AS gmv FROM orders GROUP BY week", ["week", "gmv"],
|
||||
[["2026-W42", 2050], ["2026-W41", 2220]], types={"week": "text", "gmv": "number"})
|
||||
result = await additive(snaps=[weekly(), head], headline={"base": "Q1.r1.gmv", "target": "Q1.r0.gmv"})
|
||||
assert result["verdict"] == {"code": "unclosed", "subjects": [], "direction": None}
|
||||
assert result["values"]["gap"] == "24" and result["values"]["top_share"] is None
|
||||
assert all(r[5] is None for r in result["table"]["rows"]) # 不给占比
|
||||
assert result["phrase"] == "按地区拆开后各段之和和总数对不上(差 24),不判断变化来自哪里"
|
||||
assert [i["role"] for i in result["inputs"]] == ["headline_base", "headline_target", "data"]
|
||||
assert result["inputs"][0] == {"ref": "Q1.r1.gmv", "artifact": head["artifact"], "role": "headline_base",
|
||||
"row": 1, "column": "gmv", "value": "2220"}
|
||||
assert "各段之和要和总数对得上" in result["criterion"]
|
||||
|
||||
|
||||
async def test_a_rounded_headline_closes_within_its_written_precision():
|
||||
rows = [["华东", "2026-W41", "820.25"], ["华东", "2026-W42", "700.12"], ["华北", "2026-W41", "1399.9"],
|
||||
["华北", "2026-W42", "1325.84"]]
|
||||
for written, closes in (("2220.2", True), ("2220", True), ("2221", False)):
|
||||
head = snap("Q1", "SELECT SUM(amount) AS gmv FROM orders", ["gmv_base", "gmv"], [[written, "2025.96"]])
|
||||
result = await additive(rows, snaps=[weekly(rows), head], headline=["Q1.r0.gmv_base", "Q1.r0.gmv"])
|
||||
assert (result["verdict"]["code"] != "unclosed") is closes, written
|
||||
|
||||
|
||||
async def test_drilling_down_checks_closure_against_the_parent_segment():
|
||||
top = await additive()
|
||||
province = snap("Q4", "SELECT province, week, SUM(amount) AS gmv FROM orders WHERE region = '华东' GROUP BY 1, 2",
|
||||
["province", "week", "gmv"],
|
||||
[["沪", "2026-W41", 500], ["沪", "2026-W42", 410], ["苏", "2026-W41", 320], ["苏", "2026-W42", 290]])
|
||||
params = {"query": "Q4", "by": ["province"], "period": "week", "base": "2026-W41", "target": "2026-W42",
|
||||
"measure": "gmv", "parent": {"of": "A3", "key": ["华东"]}}
|
||||
child = await analyze("decompose", "additive", snaps=[province], parents={"A3": top}, **params)
|
||||
assert child["verdict"]["code"] == "driver" and child["verdict"]["subjects"] == [["沪"]]
|
||||
assert child["params"]["depth"] == 1 and child["params"]["parent"] == {"of": "A3", "key": ["华东"]}
|
||||
assert child["inputs"][0]["role"] == "parent" and child["inputs"][0]["ref"] == "A3"
|
||||
assert "max_drill_depth" in child["settings"]
|
||||
assert {c["code"] for c in child["caveats"]} == set() # 有 parent 就不是「以各段之和为总数」
|
||||
short = snap("Q4", province["sql"], ["province", "week", "gmv"],
|
||||
[["沪", "2026-W41", 500], ["沪", "2026-W42", 410], ["苏", "2026-W41", 300], ["苏", "2026-W42", 290]])
|
||||
unclosed = await analyze("decompose", "additive", snaps=[short], parents={"A3": top}, **params)
|
||||
assert unclosed["verdict"]["code"] == "unclosed" and unclosed["values"]["gap"] == "20"
|
||||
# 超深:上一层已经是第 1 层,最多 1 层
|
||||
error = await call_error("decompose", "additive", snaps=[province], parents={"A3": child},
|
||||
settings={"max_drill_depth": 1}, **{**params, "parent": {"of": "A3", "key": ["沪"]}})
|
||||
assert error.code == "too_deep" and "最多 1 层" in error.message
|
||||
missing = await call_error("decompose", "additive", snaps=[province], parents={"A3": top},
|
||||
**{**params, "parent": {"of": "A3", "key": ["东北"]}})
|
||||
assert missing.code == "parent_key_missing"
|
||||
absent = await call_error("decompose", "additive", snaps=[province], **params)
|
||||
assert absent.code == "parent_missing"
|
||||
|
||||
|
||||
async def test_new_and_vanished_segments_and_empty_cells_are_flagged():
|
||||
rows = [["华东", "2026-W41", 800], ["华东", "2026-W42", 700], ["西南", "2026-W42", 90],
|
||||
["东北", "2026-W41", 40], ["华北", "2026-W41", None], ["华北", "2026-W42", 30]]
|
||||
result = await additive(rows)
|
||||
flags = {r[0]: r[6] for r in result["table"]["rows"]}
|
||||
assert flags == {"华东": None, "西南": "新出现", "东北": "消失", "华北": "为空"}
|
||||
null = next(c for c in result["caveats"] if c["code"] == "null_values")
|
||||
assert null["text"] == "华北的销售额为空,按 0 算"
|
||||
assert next(r for r in result["table"]["rows"] if r[0] == "西南")[4] is None # 基期为 0 不给变化率
|
||||
|
||||
|
||||
async def test_two_dimension_keys():
|
||||
sql = "SELECT region, channel, week, SUM(amount) AS gmv FROM orders GROUP BY 1, 2, 3"
|
||||
rows = [["华东", "线上", "W1", 500], ["华东", "线上", "W2", 300], ["华东", "线下", "W1", 300], ["华东", "线下", "W2", 310],
|
||||
["华北", "线上", "W1", 200], ["华北", "线上", "W2", 205]]
|
||||
s = snap("Q2", sql, ["region", "channel", "week", "gmv"], rows)
|
||||
result = await analyze("decompose", "additive", snaps=[s], query="Q2", by=["region", "channel"], period="week",
|
||||
base="W1", target="W2", measure="gmv")
|
||||
assert result["verdict"]["subjects"] == [["华东", "线上"]]
|
||||
assert "主要来自华东·线上" in result["phrase"]
|
||||
|
||||
|
||||
async def test_ratio_and_non_additive_measures_are_call_errors():
|
||||
avg = snap("Q2", "SELECT region, week, AVG(amount) AS aov FROM orders GROUP BY 1, 2", ["region", "week", "aov"], WEEKLY)
|
||||
error = await call_error("decompose", "additive", snaps=[avg], query="Q2", by="region", period="week",
|
||||
base="2026-W41", target="2026-W42", measure="aov")
|
||||
assert error.code == "ratio_as_additive" and "改用比率拆解" in error.message
|
||||
rate_sql = "SELECT region, week, SUM(refund_rate) AS r FROM orders GROUP BY 1, 2"
|
||||
labeled = {"shop": {"orders": notes(columns={"refund_rate": {"measure": "ratio"}})}}
|
||||
error = await call_error("decompose", "additive", snaps=[snap("Q2", rate_sql, ["region", "week", "r"], WEEKLY)],
|
||||
catalogs=labeled, query="Q2", by="region", period="week", base="2026-W41",
|
||||
target="2026-W42", measure="r")
|
||||
assert error.code == "ratio_as_additive"
|
||||
# 只是推断(没人确认)的照算,写附带说明
|
||||
guessed = {"shop": {"orders": notes(columns={"refund_rate": {"measure": {"value": "ratio", "source": "llm",
|
||||
"status": "proposed"}}})}}
|
||||
result = await analyze("decompose", "additive", snaps=[snap("Q2", rate_sql, ["region", "week", "r"], WEEKLY)],
|
||||
catalogs=guessed, query="Q2", by="region", period="week", base="2026-W41",
|
||||
target="2026-W42", measure="r")
|
||||
note = next(c for c in result["caveats"] if c["code"] == "measure_unlabeled")
|
||||
assert note["text"] == "目录里「r」是比率只是推断,没人确认,按可以按段相加处理"
|
||||
unlabeled = await additive(snaps=[weekly()], catalogs={})
|
||||
assert any(c["code"] == "measure_unlabeled" and "目录没标「gmv」的度量类型" in c["text"] for c in unlabeled["caveats"])
|
||||
|
||||
|
||||
async def test_call_errors_tell_the_agent_how_to_fix_the_call():
|
||||
dup = WEEKLY + [["华东", "2026-W42", 5]]
|
||||
error = await call_error("decompose", "additive", snaps=[weekly(dup)], query="Q2", by="region", period="week",
|
||||
base="2026-W41", target="2026-W42", measure="gmv")
|
||||
assert error.code == "duplicate_rows" and "在 SQL 里按 地区、week 汇总" in error.message
|
||||
masked = await call_error("decompose", "additive", snaps=[weekly(masked=["Region"])], query="Q2", by="region",
|
||||
period="week", base="2026-W41", target="2026-W42", measure="gmv")
|
||||
assert masked.code == "masked_column"
|
||||
missing = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="region", period="week",
|
||||
base="2026-W40", target="2026-W42", measure="gmv")
|
||||
assert missing.code == "period_not_found" and "2026-W41" in missing.message
|
||||
column = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="area", period="week",
|
||||
base="2026-W41", target="2026-W42", measure="gmv")
|
||||
assert column.code == "unknown_column" and "region、week、gmv" in column.message
|
||||
text = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="gmv", period="week",
|
||||
base="2026-W41", target="2026-W42", measure="region")
|
||||
assert text.code == "not_numeric"
|
||||
unknown = await call_error("decompose", "additive", snaps=[weekly()], query="Q9", by="region", period="week",
|
||||
base="2026-W41", target="2026-W42", measure="gmv")
|
||||
assert unknown.code == "unknown_query"
|
||||
same_column = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="week", period="week",
|
||||
base="2026-W41", target="2026-W42", measure="gmv")
|
||||
assert same_column.code == "bad_by"
|
||||
|
||||
|
||||
async def test_truncated_snapshots_and_single_segments_are_refused():
|
||||
result = await additive(snaps=[weekly(truncated=True)])
|
||||
assert result["status"] == "refused" and result["reason_code"] == "truncated"
|
||||
assert result["reason"] == "Q2 只取回了前 8 行,各段之和不是完整的和"
|
||||
assert result["phrase"] == "变化拆解没有做成:Q2 只取回了前 8 行,各段之和不是完整的和"
|
||||
assert result["verdict"] is None and result["table"] is None and result["values"] == {}
|
||||
one = await additive([["华东", "2026-W41", 1], ["华东", "2026-W42", 2]])
|
||||
assert one["reason_code"] == "too_few_segments"
|
||||
|
||||
|
||||
async def test_two_snapshots_check_comparability_first():
|
||||
base = snap("Q1", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' AND "
|
||||
"order_date < '2026-10-12' GROUP BY region", ["region", "gmv"],
|
||||
[["华东", 820], ["华北", 600], ["华南", 700]])
|
||||
target = snap("Q2", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-12' AND "
|
||||
"order_date < '2026-10-19' GROUP BY region", ["region", "gmv"],
|
||||
[["华东", 700], ["华北", 560], ["华南", 666]])
|
||||
probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")})
|
||||
result = await analyze("decompose", "additive", snaps=[base, target], probe=probe, base_query="Q1",
|
||||
target_query="Q2", by="region", measure="gmv")
|
||||
assert result["verdict"]["code"] == "driver" and result["params"]["today"] == "2026-10-21"
|
||||
assert not any(c["code"] == "comparability" for c in result["caveats"])
|
||||
assert [i["role"] for i in result["inputs"]] == ["base", "target"]
|
||||
assert result["probes"] == [{"source": "shop", "table": "orders", "column": "order_date", "min": "2025-01-01",
|
||||
"max": "2026-10-20", "basis": "live", "at": "2026-10-21T09:00:00"}]
|
||||
# 本期没过完:照样拆,判定后附上可比性结论
|
||||
partial = snap("Q2", target["sql"].replace("'2026-10-19'", "'2026-10-25'"), ["region", "gmv"], target["rows"])
|
||||
result = await analyze("decompose", "additive", snaps=[base, partial], probe=probe, base_query="Q1",
|
||||
target_query="Q2", by="region", measure="gmv")
|
||||
note = next(c for c in result["caveats"] if c["code"] == "comparability")
|
||||
assert note["text"].startswith("两期不可比:") and "本期数据只到 2026-10-20,还差 4 天" in note["text"]
|
||||
assert result["verdict"]["code"] == "driver" and note["text"] in result["phrase"]
|
||||
# 口径结构不同:不拆
|
||||
other = snap("Q2", "SELECT region, SUM(amount) AS gmv FROM orders JOIN refunds USING (order_id) WHERE order_date "
|
||||
">= '2026-10-12' GROUP BY region", ["region", "gmv"], target["rows"])
|
||||
refused = await analyze("decompose", "additive", snaps=[base, other], probe=probe, base_query="Q1",
|
||||
target_query="Q2", by="region", measure="gmv")
|
||||
assert refused["reason_code"] == "incomparable_sides" and refused["reason"].startswith("两份查询的口径结构不同")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# ratio
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
RATIO_SQL = "SELECT channel, week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY 1, 2"
|
||||
|
||||
|
||||
def _ratio_snap(rows):
|
||||
return snap("Q1", RATIO_SQL, ["channel", "week", "refunds", "orders"], rows)
|
||||
|
||||
|
||||
async def ratio(rows, **params):
|
||||
params = {"query": "Q1", "by": "channel", "period": "week", "base": "W41", "target": "W42",
|
||||
"numerator": "refunds", "denominator": "orders", **params}
|
||||
return await analyze("decompose", "ratio", snaps=[_ratio_snap(rows)], **params)
|
||||
|
||||
|
||||
def test_mix_plus_rate_effects_equal_the_ratio_change_exactly():
|
||||
rng = random.Random(11)
|
||||
for _ in range(200):
|
||||
segs = []
|
||||
for _ in range(rng.randint(2, 7)):
|
||||
d_b, d_t = rng.choice([0, rng.randint(1, 5000)]), rng.randint(1, 5000)
|
||||
segs.append((Fraction(rng.randint(0, d_b)) if d_b else Fraction(0), Fraction(d_b),
|
||||
Fraction(rng.randint(0, d_t)), Fraction(d_t)))
|
||||
if sum(s[1] for s in segs) == 0:
|
||||
continue
|
||||
R, R2, effects = ratio_effects(segs)
|
||||
assert sum((e[4] for e in effects), Fraction(0)) + sum((e[5] for e in effects), Fraction(0)) == R2 - R
|
||||
assert sum((e[0] for e in effects), Fraction(0)) == 1 == sum((e[1] for e in effects), Fraction(0))
|
||||
|
||||
|
||||
async def test_simpson_each_channel_improves_but_the_mix_makes_the_whole_worse():
|
||||
rows = [["线上", "W41", 50, 500], ["线上", "W42", 90, 1000], ["线下", "W41", 10, 1000], ["线下", "W42", 4, 500]]
|
||||
result = await ratio(rows, label="退款率")
|
||||
assert result["verdict"] == {"code": "mix_driven", "subjects": [["线上"]], "direction": "up"}
|
||||
assert result["phrase"].startswith("退款率上升 2.3 个百分点(4.0% 到 6.3%),主要来自结构变化:线上占比从 33.3% 到 66.7%")
|
||||
simpson = next(c for c in result["caveats"] if c["code"] == "simpson")
|
||||
assert simpson["text"] == "各段的退款率大多在下降,整体却上升:变化来自结构,线上占比从 33.3% 升到 66.7%"
|
||||
values = result["values"]
|
||||
assert Fraction(values["mix_total"]) + Fraction(values["rate_total"]) == pytest.approx(Fraction(values["delta"]))
|
||||
assert result["table"]["columns"][:5] == ["channel", "n_base", "n_target", "d_base", "d_target"]
|
||||
|
||||
|
||||
async def test_rate_driven_both_and_small_denominators():
|
||||
rows = [["线上", "W41", 50, 1000], ["线上", "W42", 80, 1000], ["线下", "W41", 20, 1000], ["线下", "W42", 26, 1000]]
|
||||
result = await ratio(rows, label="退款率")
|
||||
assert result["verdict"]["code"] == "rate_driven" and result["verdict"]["subjects"] == [["线上"]]
|
||||
assert result["phrase"] == "退款率上升 1.8 个百分点(3.5% 到 5.3%),主要来自各段自身的变化,线上影响最大"
|
||||
tiny = [["线上", "W41", 2, 20], ["线上", "W42", 3, 25], ["线下", "W41", 20, 1000], ["线下", "W42", 30, 1000]]
|
||||
small = await ratio(tiny, label="退款率")
|
||||
note = next(c for c in small["caveats"] if c["code"] == "small_denominator")
|
||||
assert note["text"] == "线上的 orders 不到 30,比率波动大,不参与多数判断"
|
||||
assert "分母小" in next(r for r in small["table"]["rows"] if r[0] == "线上")[-1]
|
||||
|
||||
|
||||
async def test_ratio_refusals():
|
||||
zero = [["线上", "W41", 0, 0], ["线上", "W42", 3, 100], ["线下", "W41", 0, 0], ["线下", "W42", 2, 100]]
|
||||
result = await ratio(zero)
|
||||
assert result["reason_code"] == "zero_base" and result["reason"] == "基期的 orders 合计为 0,算不了比率"
|
||||
orphan = [["线上", "W41", 5, 0], ["线上", "W42", 3, 100], ["线下", "W41", 1, 100], ["线下", "W42", 2, 100]]
|
||||
assert (await ratio(orphan))["reason_code"] == "zero_base"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# factor
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _factor_snap(rows):
|
||||
return snap("Q1", "SELECT week, SUM(amount) AS gmv, COUNT(*) AS orders, ROUND(AVG(amount), 2) AS aov FROM orders "
|
||||
"GROUP BY week", ["week", "gmv", "orders", "aov"], rows)
|
||||
|
||||
|
||||
async def factor(rows, **params):
|
||||
params = {"query": "Q1", "period": "week", "base": "W41", "target": "W42", "product": "gmv",
|
||||
"factors": ["orders", "aov"], "label": "销售额", **params}
|
||||
return await analyze("decompose", "factor", snaps=[_factor_snap(rows)], **params)
|
||||
|
||||
|
||||
def test_lmdi_contributions_add_up_to_the_change_within_float_error():
|
||||
rng = random.Random(3)
|
||||
for _ in range(200):
|
||||
k = rng.randint(2, 4)
|
||||
base = [rng.uniform(0.5, 500) for _ in range(k)]
|
||||
target = [b * rng.uniform(0.5, 1.8) for b in base]
|
||||
m_b = math.prod(base) * (1 + rng.uniform(-1e-6, 1e-6)) # 取整过的因子:乘积差一点点
|
||||
m_t = math.prod(target) * (1 + rng.uniform(-1e-6, 1e-6))
|
||||
contrib = lmdi(base, target, m_b, m_t)
|
||||
assert math.fsum(contrib) == pytest.approx(m_t - m_b, rel=1e-9, abs=1e-9 * max(m_b, m_t))
|
||||
same = lmdi([2.0, 3.0], [2.0, 3.0], 6.0, 6.0)
|
||||
assert same == [0.0, 0.0]
|
||||
|
||||
|
||||
async def test_factor_driver_with_rounded_factors():
|
||||
rows = [["W41", 1000, 137, "7.30"], ["W42", 1320, 160, "8.25"]]
|
||||
result = await factor(rows)
|
||||
assert result["verdict"]["code"] in ("driver", "both")
|
||||
contrib = [Fraction(r[4]) for r in result["table"]["rows"]]
|
||||
assert float(sum(contrib)) == pytest.approx(320, rel=1e-9)
|
||||
clean = await factor([["W41", "1000", 100, "10"], ["W42", "1320", 120, "11"]])
|
||||
assert clean["verdict"] == {"code": "driver", "subjects": [["orders"]], "direction": "up"}
|
||||
assert clean["phrase"].startswith("销售额上升 320(+32.0%),主要来自 orders 的变化(贡献 +")
|
||||
assert clean["table"]["columns"] == ["factor", "base", "target", "pct", "contrib", "share"]
|
||||
|
||||
|
||||
async def test_factor_call_errors_and_refusals():
|
||||
# 客单价写到两位小数:容差是它的半个末位(加上订单数的),差 1.5% 不是取整造成的
|
||||
error = await call_error("decompose", "factor", snaps=[_factor_snap([["W41", 1000, 100, "10.00"],
|
||||
["W42", 1300, 120, "11.00"]])],
|
||||
query="Q1", period="week", base="W41", target="W42", product="gmv",
|
||||
factors=["orders", "aov"])
|
||||
assert error.code == "not_product" and "本期差" in error.message
|
||||
negative = await factor([["W41", 0, 0, 10], ["W42", 1320, 120, 11]])
|
||||
assert negative["reason_code"] == "non_positive"
|
||||
flat = await factor([["W41", 1000, 100, 10], ["W42", 1000, 125, 8]])
|
||||
assert flat["verdict"]["code"] == "flat"
|
||||
few = await call_error("decompose", "factor", snaps=[_factor_snap([["W41", 1, 1, 1], ["W42", 1, 1, 1]])],
|
||||
query="Q1", period="week", base="W41", target="W42", product="gmv", factors=["orders"])
|
||||
assert few.code == "bad_factors"
|
||||
|
||||
|
||||
async def test_ratio_headline_closure_and_drilling_into_a_channel():
|
||||
rows = [["线上", "W41", 50, 1000], ["线上", "W42", 80, 1000], ["线下", "W41", 20, 1000], ["线下", "W42", 26, 1000]]
|
||||
head = snap("Q9", "SELECT week, SUM(refund_flag) * 1.0 / COUNT(*) AS rate FROM orders GROUP BY week",
|
||||
["week", "rate"], [["W41", "0.035"], ["W42", "0.053"]])
|
||||
params = {"query": "Q1", "by": "channel", "period": "week", "base": "W41", "target": "W42",
|
||||
"numerator": "refunds", "denominator": "orders", "label": "退款率"}
|
||||
closed = await analyze("decompose", "ratio", snaps=[_ratio_snap(rows), head],
|
||||
headline={"base": "Q9.r0.rate", "target": "Q9.r1.rate"}, **params)
|
||||
assert closed["verdict"]["code"] == "rate_driven"
|
||||
off = snap("Q9", head["sql"], ["week", "rate"], [["W41", "0.035"], ["W42", "0.06"]])
|
||||
unclosed = await analyze("decompose", "ratio", snaps=[_ratio_snap(rows), off],
|
||||
headline={"base": "Q9.r0.rate", "target": "Q9.r1.rate"}, **params)
|
||||
assert unclosed["verdict"]["code"] == "unclosed"
|
||||
assert unclosed["phrase"].startswith("按渠道拆开后各段合起来的比率和总数对不上")
|
||||
# 线上按地区下钻:各地区的分子、分母之和要等于上一层线上那一段的
|
||||
region = snap("Q2", "SELECT region, week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders "
|
||||
"WHERE channel = '线上' GROUP BY 1, 2", ["region", "week", "refunds", "orders"],
|
||||
[["华东", "W41", 30, 600], ["华东", "W42", 55, 600], ["华北", "W41", 20, 400], ["华北", "W42", 25, 400]])
|
||||
child = await analyze("decompose", "ratio", snaps=[region], parents={"A1": closed},
|
||||
**{**params, "query": "Q2", "by": "region", "parent": {"of": "A1", "key": ["线上"]}})
|
||||
assert child["verdict"]["code"] != "unclosed" and child["params"]["depth"] == 1
|
||||
wrong = await call_error("decompose", "additive", snaps=[weekly()], parents={"A1": closed}, query="Q2",
|
||||
by="region", period="week", base="2026-W41", target="2026-W42", measure="gmv",
|
||||
parent={"of": "A1", "key": ["线上"]})
|
||||
assert wrong.code == "parent_not_decompose"
|
||||
|
||||
|
||||
async def test_the_same_query_cannot_be_both_periods():
|
||||
error = await call_error("decompose", "additive", snaps=[weekly()], base_query="Q2", target_query="Q2",
|
||||
by="region", measure="gmv")
|
||||
assert error.code == "same_query"
|
||||
error = await call_error("comparable", snaps=[weekly()], base_query="Q2", target_query="Q2")
|
||||
assert error.code == "same_query"
|
||||
@@ -0,0 +1,138 @@
|
||||
"""差异检验(app/analysis/differ.py,细案 01 §5):比率小样本走 Fisher、均值 Welch、Holm、rows 敏感性翻转。每个用例都顺带核对
|
||||
复算和运行逐字段相同。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
|
||||
from app.analysis import stats
|
||||
from tests.analysis_samples import analyze, call_error, snap
|
||||
|
||||
RATE_SQL = "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region"
|
||||
|
||||
|
||||
def rates(rows):
|
||||
return snap("Q1", RATE_SQL, ["region", "refunds", "orders"], rows)
|
||||
|
||||
|
||||
async def rate(rows, **params):
|
||||
params = {"query": "Q1", "by": "region", "numerator": "refunds", "denominator": "orders", "label": "退款率", **params}
|
||||
return await analyze("differ", "rate", snaps=[rates(rows)], **params)
|
||||
|
||||
|
||||
async def test_small_samples_use_fisher_and_say_it_may_be_noise():
|
||||
result = await rate([["西南", 2, 40], ["华东", 5, 230]])
|
||||
assert result["method"] == "differ.rate@1"
|
||||
assert result["verdict"] == {"code": "no_evidence", "subjects": [["西南"], ["华东"]], "direction": None}
|
||||
assert result["values"]["test"] == "fisher" and result["values"]["n_a"] == "40" and result["values"]["n_b"] == "230"
|
||||
assert Fraction(result["values"]["p"]) == pytest.approx(stats.fisher_exact(2, 38, 5, 225), rel=1e-9)
|
||||
assert result["phrase"] == ("西南和华东的退款率差异可能只是样本波动(样本量 40、230);样本少(有格子的期望频数不到 5),"
|
||||
"用了精确检验")
|
||||
assert "p 值" not in result["phrase"] and "显著" not in result["phrase"]
|
||||
assert result["settings"] == {"differ_alpha": 0.05, "min_expected": 5}
|
||||
|
||||
|
||||
async def test_large_samples_use_the_pooled_z_test():
|
||||
result = await rate([["西南", 80, 1000], ["华东", 40, 1000]])
|
||||
assert result["verdict"] == {"code": "differs", "subjects": [["西南"], ["华东"]], "direction": "up"}
|
||||
z, p = stats.pooled_z(80, 1000, 40, 1000)
|
||||
assert Fraction(result["values"]["p"]) == pytest.approx(p, rel=1e-9) and result["values"]["test"] == "z"
|
||||
low, high = stats.newcombe(80, 1000, 40, 1000)
|
||||
assert Fraction(result["values"]["ci_lo"]) == pytest.approx(low, rel=1e-9)
|
||||
assert result["values"]["a"] == "0.08" and result["values"]["diff"] == "0.04"
|
||||
assert result["phrase"] == "西南的退款率(8.0%)高于华东(4.0%),差异站得住"
|
||||
lower = await rate([["西南", 80, 1000], ["华东", 40, 1000]], pairs=[["华东", "西南"]])
|
||||
assert lower["verdict"]["direction"] == "down" and lower["phrase"].startswith("华东的退款率(4.0%)低于西南(8.0%)")
|
||||
|
||||
|
||||
async def test_several_groups_each_against_the_rest_with_holm():
|
||||
rows = [["西南", 80, 1000], ["华东", 40, 1000], ["华南", 42, 1000], ["华北", 38, 1000]]
|
||||
result = await rate(rows)
|
||||
table = result["table"]
|
||||
assert [r[0] for r in table["rows"]] == ["西南", "华东", "华南", "华北"]
|
||||
assert table["rows"][0][1] == "其余各地区"
|
||||
raw = [Fraction(r[10]) for r in table["rows"]]
|
||||
adjusted = [Fraction(r[11]) for r in table["rows"]]
|
||||
assert all(a >= p for a, p in zip(adjusted, raw))
|
||||
assert adjusted == pytest.approx(stats.holm([float(p) for p in raw]), rel=1e-9)
|
||||
assert result["verdict"]["code"] == "differs" and result["verdict"]["subjects"] == [["西南"], ["其余各地区"]]
|
||||
assert any(c["code"] == "multiple_comparisons" and c["text"] == "比了 4 对,已按次数校正" for c in result["caveats"])
|
||||
assert result["phrase"] == "比了 4 对,1 对差异站得住:西南的退款率(8.0%)高于其余各地区(4.0%)"
|
||||
ref = await rate(rows, reference="华东")
|
||||
assert [r[0] for r in ref["table"]["rows"]] == ["西南", "华南", "华北"] and ref["params"]["reference"] == "华东"
|
||||
|
||||
|
||||
async def test_rate_call_errors():
|
||||
assert (await call_error("differ", "rate", snaps=[rates([["西南", 5, 4], ["华东", 1, 9]])], query="Q1", by="region",
|
||||
numerator="refunds", denominator="orders")).code == "not_count"
|
||||
assert (await call_error("differ", "rate", snaps=[rates([["西南", 1.5, 4], ["华东", 1, 9]])], query="Q1",
|
||||
by="region", numerator="refunds", denominator="orders")).code == "not_count"
|
||||
unknown = await call_error("differ", "rate", snaps=[rates([["西南", 1, 4], ["华东", 1, 9]])], query="Q1",
|
||||
by="region", numerator="refunds", denominator="orders", pairs=[["西南", "东北"]])
|
||||
assert unknown.code == "unknown_group" and "西南、华东" in unknown.message
|
||||
dup = await call_error("differ", "rate", snaps=[rates([["西南", 1, 4], ["西南", 1, 9]])], query="Q1", by="region",
|
||||
numerator="refunds", denominator="orders")
|
||||
assert dup.code == "duplicate_rows"
|
||||
|
||||
|
||||
async def test_means_from_sums_and_squares_use_welch():
|
||||
sql = "SELECT channel, SUM(amount) AS s, SUM(amount * amount) AS ss, COUNT(*) AS n FROM orders GROUP BY channel"
|
||||
rows = [["线上", 5000, 300000, 100], ["线下", 4000, 200000, 100]]
|
||||
result = await analyze("differ", "mean", snaps=[snap("Q1", sql, ["channel", "s", "ss", "n"], rows)], query="Q1",
|
||||
by="channel", sum="s", sumsq="ss", n="n", label="客单价")
|
||||
va = (300000 - 5000 ** 2 / 100) / 99
|
||||
vb = (200000 - 4000 ** 2 / 100) / 99
|
||||
expected = stats.welch(50, va, 100, 40, vb, 100)
|
||||
assert Fraction(result["values"]["p"]) == pytest.approx(expected.p, rel=1e-9)
|
||||
assert result["verdict"]["code"] == "differs" and result["values"]["test"] == "welch"
|
||||
assert result["phrase"] == "线上的客单价(50)高于线下(40),差异站得住"
|
||||
assert result["method"] == "differ.welch@1" and result["settings"] == {"differ_alpha": 0.05, "min_n": 30}
|
||||
|
||||
|
||||
async def test_means_from_mean_sd_and_n_and_small_samples():
|
||||
sql = "SELECT channel, avg_amount, sd_amount, n FROM channel_stats"
|
||||
rows = [["线上", 52, 12, 12], ["线下", 50, 11, 400]]
|
||||
result = await analyze("differ", "mean", snaps=[snap("Q1", sql, ["channel", "avg_amount", "sd_amount", "n"], rows)],
|
||||
query="Q1", by="channel", mean="avg_amount", sd="sd_amount", n="n", label="客单价")
|
||||
assert result["verdict"]["code"] == "no_evidence"
|
||||
note = next(c for c in result["caveats"] if c["code"] == "small_sample")
|
||||
assert note["text"] == "有一组的样本数不到 30(线上 12),结论不稳"
|
||||
assert result["phrase"].startswith("线上和线下的客单价差异可能只是样本波动(样本量 12、400)")
|
||||
|
||||
|
||||
async def test_rows_mode_flags_a_single_value_that_flips_the_verdict():
|
||||
base = [["线下", v] for v in [40, 60] * 200]
|
||||
online = [["线上", f"{v:.1f}"] for v in [41.3, 61.3] * 200]
|
||||
online[-1] = ["线上", "231.3"]
|
||||
s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], online + base)
|
||||
result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount", label="客单价")
|
||||
assert result["method"] == "differ.welch_rows@1"
|
||||
assert result["verdict"]["code"] == "differs"
|
||||
note = next(c for c in result["caveats"] if c["code"] == "outlier_sensitive")
|
||||
assert note["text"] == "去掉最大的一笔后,差异不再站得住"
|
||||
assert note["text"] in result["phrase"]
|
||||
hidden = [["线上", 50 + (i % 7)] for i in range(30)] + [["线下", 48 + (i % 5)] for i in range(30)] + [["线上", 400]]
|
||||
s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], hidden)
|
||||
flipped = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount", label="客单价")
|
||||
assert flipped["verdict"]["code"] == "no_evidence"
|
||||
assert any(c["text"] == "去掉最大的一笔后,差异变得站得住" for c in flipped["caveats"])
|
||||
|
||||
|
||||
async def test_two_periods_of_the_same_group():
|
||||
sql = "SELECT week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders WHERE region = '西南' GROUP BY week"
|
||||
s = snap("Q1", sql, ["week", "refunds", "orders"], [["W41", 30, 1000], ["W42", 60, 1000], ["W40", 31, 1000]])
|
||||
result = await analyze("differ", "rate", snaps=[s], query="Q1", period="week", base="W41", target="W42",
|
||||
numerator="refunds", denominator="orders", label="退款率")
|
||||
assert result["verdict"] == {"code": "differs", "subjects": [["W42"], ["W41"]], "direction": "up"}
|
||||
assert result["inputs"][0]["rows"] == [0, 1]
|
||||
assert result["phrase"] == "W42 的退款率(6.0%)高于 W41(3.0%),差异站得住"
|
||||
|
||||
|
||||
async def test_a_group_with_one_value_cannot_be_tested():
|
||||
s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], [["线上", 5], ["线下", 4], ["线下", 6]])
|
||||
result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount")
|
||||
assert result["verdict"]["code"] == "no_evidence" and result["values"]["p"] is None
|
||||
assert result["table"]["rows"][0][-1] == "样本不够,算不了"
|
||||
assert math.isfinite(float(result["values"]["diff"]))
|
||||
@@ -0,0 +1,57 @@
|
||||
"""期次(app/analysis/periods.py):日期、YYYY-MM、YYYY-Www、YYYY-Qn、整数序号认得出、能排序、知道覆盖哪几天;粒度按值推断;
|
||||
写法混着、带时刻的认不出。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from app.analysis import periods as pd
|
||||
|
||||
|
||||
def test_dates_infer_their_grain():
|
||||
items, grain = pd.parse(["2026-10-01", "2026-10-02", "2026-10-04"])
|
||||
assert grain == "day" and [i.start for i in items] == [date(2026, 10, 1), date(2026, 10, 2), date(2026, 10, 4)]
|
||||
items, grain = pd.parse(["2026-09-28", "2026-10-05", "2026-10-19"]) # 隔 7 天的整数倍:周(缺一周)
|
||||
assert grain == "week" and [i.ordinal for i in items] == [0, 1, 3] and items[0].end == date(2026, 10, 4)
|
||||
items, grain = pd.parse(["2026-08-01", "2026-09-01", "2026-10-01"])
|
||||
assert grain == "month" and items[1].end == date(2026, 9, 30)
|
||||
items, grain = pd.parse(["2026-01-01", "2026-04-01", "2026-07-01"])
|
||||
assert grain == "quarter" and items[0].end == date(2026, 3, 31)
|
||||
items, grain = pd.parse([20261001, 20261002])
|
||||
assert grain == "day" and items[1].start == date(2026, 10, 2)
|
||||
assert pd.parse(["2026-10-01 00:00:00", "2026-10-02 00:00:00"])[1] == "day"
|
||||
|
||||
|
||||
def test_labels_for_months_weeks_quarters_and_sequences():
|
||||
items, grain = pd.parse(["2026-09", "2026-10"])
|
||||
assert grain == "month" and items[1].ordinal - items[0].ordinal == 1 and items[1].end == date(2026, 10, 31)
|
||||
items, grain = pd.parse([202609, 202610])
|
||||
assert grain == "month"
|
||||
items, grain = pd.parse(["2026-W41", "2026-W42", "2026-W01"])
|
||||
assert grain == "week" and items[1].start == date(2026, 10, 12) and items[1].end == date(2026, 10, 18)
|
||||
assert items[1].ordinal - items[0].ordinal == 1
|
||||
items, grain = pd.parse(["2025-W52", "2026-W01"]) # 跨年也连着
|
||||
assert items[1].ordinal - items[0].ordinal == 1
|
||||
items, grain = pd.parse(["2026Q3", "2026-Q4"])
|
||||
assert grain == "quarter" and items[1].start == date(2026, 10, 1)
|
||||
items, grain = pd.parse([3, 1, 2])
|
||||
assert grain == "seq" and items[0].start is None and [i.ordinal for i in items] == [3, 1, 2]
|
||||
|
||||
|
||||
def test_unreadable_periods():
|
||||
assert pd.parse(["2026-10-01", "2026-W41"]) is None # 写法混着
|
||||
assert pd.parse(["2026-10-01 08:30:00", "2026-10-02 08:30:00"]) is None # 带时刻
|
||||
assert pd.parse(["上周", "本周"]) is None
|
||||
assert pd.parse(["2026-10", "2026-11"], "week") is None # 粒度和写法对不上
|
||||
assert pd.parse(["2026-10-01", "2026-10-03"], "week") is None # 按周但不隔 7 天
|
||||
assert pd.parse([]) is None and pd.parse([None, "2026-10-01"]) is None
|
||||
|
||||
|
||||
def test_calendar_units_and_weekend_days():
|
||||
assert pd.calendar_unit(date(2026, 9, 1), date(2026, 9, 30)) == "month"
|
||||
assert pd.calendar_unit(date(2026, 7, 1), date(2026, 9, 30)) == "quarter"
|
||||
assert pd.calendar_unit(date(2026, 1, 1), date(2026, 12, 31)) == "year"
|
||||
assert pd.calendar_unit(date(2026, 9, 2), date(2026, 9, 30)) is None
|
||||
assert pd.weekend_days(date(2026, 10, 12), date(2026, 10, 18)) == 2 # 周一到周日
|
||||
assert pd.weekend_days(date(2026, 10, 16), date(2026, 10, 18)) == 2 # 周五到周日
|
||||
assert pd.weekend_days(date(2026, 10, 12), date(2026, 10, 14)) == 0
|
||||
assert pd.weekend_days(date(2026, 10, 1), date(2026, 10, 31)) == 9
|
||||
@@ -0,0 +1,121 @@
|
||||
"""判定语、数的写法、label 消毒、判定标准(app/analysis/phrases.py、numbers.py):一种说法,只在这里定义。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fractions import Fraction
|
||||
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis.contract import DEFAULTS
|
||||
from app.analysis.numbers import canon, fixed, round_sig, to_fraction
|
||||
|
||||
|
||||
def test_canonical_decimal_strings_are_plain_and_stable():
|
||||
assert canon(Fraction(-120)) == "-120"
|
||||
assert canon("2220.00") == "2220" # 去掉多余的 0
|
||||
assert canon(0.1 + 0.2) == "0.3" # 浮点留 12 位有效数字
|
||||
assert canon(Fraction(1, 3)) == "0.333333333333"
|
||||
assert canon(Fraction(7, 16)) == "0.4375" # 除得尽的照写
|
||||
assert canon(-0.0) == "0" and canon(Fraction(0)) == "0"
|
||||
assert canon(1e-30) == "0.000000000000000000000000000001" # 不带指数
|
||||
assert canon(1e20) == "100000000000000000000"
|
||||
assert canon("123456789012345678.25") == "123456789012345678.25" # 快照里的数和它们的和差照原样
|
||||
assert canon(None) is None and canon("abc") is None and canon(True) is None
|
||||
assert to_fraction(0.1) == Fraction(1, 10) and to_fraction("45678.50") == Fraction(4567850, 100)
|
||||
assert round_sig(Fraction(2, 3), 3) == Fraction(667, 1000)
|
||||
|
||||
|
||||
def test_fixed_rounds_half_up_like_render_cell():
|
||||
assert fixed(Fraction(2675, 1000), 2) == "2.68" # 2.675 不走浮点的 round(会得 2.67)
|
||||
assert fixed(Fraction(-5, 100), 1, strip=False) == "-0.1"
|
||||
assert fixed(Fraction(-4, 100), 1, strip=False) == "0.0" # 不写 -0.0
|
||||
assert fixed(Fraction(123456789, 100), 2, group=True) == "1,234,567.89"
|
||||
|
||||
|
||||
def test_number_formats_in_phrases():
|
||||
assert ph.pct(Fraction(-87, 1000)) == "-8.7%" and ph.pct(Fraction(3, 100)) == "+3.0%"
|
||||
assert ph.pct(0) == "0.0%" and ph.pct(Fraction(6186, 10000), signed=False) == "61.9%"
|
||||
assert ph.pct(Fraction(4, 100000)) == "+0.004%" # 不是 0 的不写成 0.0%
|
||||
assert ph.amount(Fraction(1234567891, 1000)) == "1,234,567.89"
|
||||
assert ph.amount(Fraction(120)) == "120" and ph.amount(Fraction(-120), signed=True) == "-120"
|
||||
assert ph.amount(Fraction(40), signed=True) == "+40"
|
||||
assert ph.amount(Fraction(21, 1000)) == "0.021" and ph.amount(Fraction(4567, 10 ** 7)) == "0.00046"
|
||||
assert ph.change(Fraction(-194), Fraction(2220)) == "下降 194(-8.7%)"
|
||||
assert ph.change(Fraction(120), Fraction(0)) == "上升 120" # 基期为 0 不给百分比
|
||||
assert ph.change(0, 5) == "没有变化"
|
||||
assert ph.ratio_change(Fraction(21, 1000), Fraction(25, 1000), proportion=True) == "上升 0.4 个百分点(2.1% 到 2.5%)"
|
||||
assert ph.ratio_change(Fraction(82), Fraction(852, 10), proportion=False) == "上升 3.2(+3.9%)"
|
||||
assert ph.threshold_pct(0.005) == "0.5%" and ph.share_words(0.5) == "一半" and ph.share_words(0.6) == "60%"
|
||||
|
||||
|
||||
def test_labels_are_scrubbed_of_numbers_units_and_markers():
|
||||
assert ph.clean_label("销售额(同比 +30%)") == "销售额(同比)"
|
||||
assert ph.clean_label("近三成客户的退款率") == "近客户的退款率"
|
||||
assert ph.clean_label("销售额[[v:Q1.r0.gmv]]") == "销售额"
|
||||
assert ph.clean_label("¥ 销售额 3.5万") == "销售额"
|
||||
assert ph.clean_label("gmv_2026") == "gmv_"
|
||||
assert len(ph.clean_label("销售额" * 30)) == ph.LABEL_MAX
|
||||
assert ph.clean_label("30%") == "" and ph.clean_label(None) == ""
|
||||
# 消毒后为空就退回下一个来源(目录里的叫法、列名)
|
||||
assert ph.pick_label("12.5%", None, "gmv") == "gmv"
|
||||
assert ph.pick_label("", "实收金额", "gmv") == "实收金额"
|
||||
assert ph.pick_label("2026", None, "2026") == "这一列"
|
||||
|
||||
|
||||
def test_numbers_stuffed_into_a_label_never_reach_the_phrase():
|
||||
label = ph.pick_label("销售额(同比 +30%,3.5万)", "销售额", "gmv")
|
||||
phrase = ph.decompose_driver(label, ph.change(-194, 2220), "华东", "-120", Fraction(6186, 10000))
|
||||
assert "30" not in phrase and "3.5" not in phrase and "万" not in phrase
|
||||
assert phrase == "销售额(同比)下降 194(-8.7%),主要来自华东(-120,占 61.9%)"
|
||||
|
||||
|
||||
def test_spacing_between_chinese_and_ascii():
|
||||
assert ph.say("gmv", "下降 194") == "gmv 下降 194"
|
||||
assert ph.say("主要来自", "online", "(", "-120") == "主要来自 online(-120"
|
||||
assert ph.say("销售额", "下降") == "销售额下降"
|
||||
assert ph.key_text(["华东", "线上"]) == "华东·线上" and ph.key_text([None]) == "(空)"
|
||||
assert ph.names(["华东", "华北", "华南"], 2) == "华东、华北等 3 段"
|
||||
|
||||
|
||||
def test_phrase_templates():
|
||||
assert ph.decompose_concentrated("销售额", "下降 194(-8.7%)", "华东、华北", Fraction(85, 100)) == \
|
||||
"销售额下降 194(-8.7%),集中在华东、华北(合计占 85.0%)"
|
||||
assert ph.decompose_dispersed("销售额", "下降 194(-8.7%)", "地区", 0.5) == \
|
||||
"销售额下降 194(-8.7%),分散在各地区,没有哪一段占到一半"
|
||||
assert ph.decompose_dispersed("销售额", "下降 194(-8.7%)", "地区", 0.6).endswith("没有哪一段占到 60%")
|
||||
assert ph.decompose_flat("销售额", Fraction(2, 1000), ("华东", "+120", "华北", "-118")) == \
|
||||
"销售额基本持平(+0.2%),内部此消彼长:华东 +120、华北 -118"
|
||||
assert ph.decompose_unclosed("地区", Fraction(-12), ratio=False) == "按地区拆开后各段之和和总数对不上(差 12),不判断变化来自哪里"
|
||||
assert ph.baseline_phrase("outside_band", "销售额", "本期变化 -8.7%", 12, "周", "-6.1% 至 +5.3%") == \
|
||||
"销售额本期变化 -8.7%,超出过去 12 周的常规波动范围(-6.1% 至 +5.3%)"
|
||||
assert ph.trend_phrase("rising", "线上销售额", 10, "周", "周", "+3.0%", (4, "up")) == \
|
||||
"过去 10 周线上销售额呈上升趋势(每周约 +3.0%),已连续 4 周上升"
|
||||
assert ph.trend_phrase("no_trend", "销售额", 8, "个月", "月", None, None) == "过去 8 个月销售额没有明显的上升或下降趋势"
|
||||
assert ph.differ_phrase([ph.differ_pair("西南", "华东", "退款率", "5.0%", "2.1%", True)], 1) == \
|
||||
"西南的退款率(5.0%)高于华东(2.1%),差异站得住"
|
||||
assert ph.no_evidence_phrase("西南", "华东", "退款率", ("40", "230"), 1) == "西南和华东的退款率差异可能只是样本波动(样本量 40、230)"
|
||||
assert ph.comparable_phrase("incomparable", ["本期数据只到 2026-10-15,还差 3 天"], []) == \
|
||||
"两期不可比:本期数据只到 2026-10-15,还差 3 天"
|
||||
assert ph.comparable_phrase("comparable", [], ["整期"]) == "两期可比,整期核对不了"
|
||||
assert ph.refused("baseline", "只有 5 期历史,至少要 8 期") == "波动基线没有做成:只有 5 期历史,至少要 8 期"
|
||||
assert ph.current_text(Fraction(-31, 1000), basis="change", lag=7, grain="day") == "比上周同一天变化 -3.1%"
|
||||
assert ph.current_text(Fraction(1, 10), basis="change", lag=52, grain="week") == "同比变化 +10.0%"
|
||||
|
||||
|
||||
def test_caveats_follow_the_phrase_at_most_two_in_order():
|
||||
caveats = [ph.caveat("measure_unlabeled", "目录没标"), ph.caveat("null_values", "西南为空"),
|
||||
ph.caveat("offset", "互相抵消"), ph.caveat("comparability", "两期有条件可比:周末不同")]
|
||||
assert ph.with_caveats("判定", caveats) == "判定;两期有条件可比:周末不同;互相抵消"
|
||||
assert ph.with_caveats("判定", caveats, skip=("offset",)) == "判定;两期有条件可比:周末不同;西南为空"
|
||||
assert ph.with_caveats("判定", [ph.caveat("headline_basis", "以各段之和为总数")]) == "判定"
|
||||
|
||||
|
||||
def test_criteria_are_written_with_the_actual_thresholds():
|
||||
text = ph.criterion_additive(DEFAULTS, top_k_max=2, summed=True)
|
||||
assert text == ("一段占总变化的一半以上算主要来源,最多前 2 段合计占 80% 以上算集中,总变化不到基期的 0.5% 算基本持平,"
|
||||
"以各段之和为总数")
|
||||
custom = {**DEFAULTS, "main_share": 0.6, "flat_pct": 0.01}
|
||||
assert "占总变化的 60% 以上" in ph.criterion_additive(custom, top_k_max=3, summed=False)
|
||||
assert "1% 算基本持平" in ph.criterion_additive(custom, top_k_max=3, summed=False)
|
||||
assert "差不到 2 倍稳健标准差" in ph.criterion_baseline(DEFAULTS, 12, "周")
|
||||
assert "3 倍以上算明显超出" in ph.criterion_baseline(DEFAULTS, 12, "周")
|
||||
assert "小于 0.05" in ph.criterion_trend(DEFAULTS, 10, "周")
|
||||
assert "Fisher" in ph.criterion_differ(DEFAULTS, "rate") and "Welch" in ph.criterion_differ(DEFAULTS, "mean")
|
||||
@@ -0,0 +1,129 @@
|
||||
"""可比性检查的现查(app/analysis/probe.py):make_prober 在真的数据源上探表的日期范围、在库里算按运行当天现算的界,遮罩列
|
||||
不探、超时限不探、现查不成退回剖析事实;Book 先登记、查完再算,结论只记最后真用到的。库是现造的 SQLite 文件,表名、列名、
|
||||
数字都是虚构的。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import uuid
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app import analysis
|
||||
from app.analysis.contract import effective_settings, same
|
||||
from app.analysis.probe import Book, make_prober
|
||||
from app.data import introspect
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from tests.analysis_samples import CATALOGS, recomputed, snap
|
||||
|
||||
CLOCK = "2026-10-21T09:30:00"
|
||||
#: SQLite 的 date('now') 按 UTC 算:表里的数据造到 UTC 的昨天
|
||||
UTC_TODAY = datetime.now(timezone.utc).date()
|
||||
LAST_DAY = UTC_TODAY - timedelta(days=1)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def shop(tmp_path):
|
||||
"""orders:order_date 从 2026-01-01 到昨天(UTC),每天一单;customers 的 signup_date、phone 是遮罩列。"""
|
||||
path = tmp_path / f"{uuid.uuid4().hex[:8]}.db"
|
||||
db = sqlite3.connect(path)
|
||||
db.executescript("CREATE TABLE orders (order_date TEXT, region TEXT, amount INTEGER);"
|
||||
"CREATE TABLE customers (signup_date TEXT, phone TEXT);"
|
||||
"INSERT INTO customers VALUES ('2026-01-05', '13900000000');")
|
||||
day = date(2026, 1, 1)
|
||||
while day <= LAST_DAY:
|
||||
db.execute("INSERT INTO orders VALUES (?, '华东', 100)", (day.isoformat(),))
|
||||
day += timedelta(days=1)
|
||||
db.commit()
|
||||
db.close()
|
||||
name = f"shop_{uuid.uuid4().hex[:8]}"
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True,
|
||||
options={"mask_columns": ["phone", "signup_date"]})
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
row.schema_cache = await introspect.introspect(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
yield row
|
||||
await engines.invalidate(row.id)
|
||||
|
||||
|
||||
async def test_the_live_range_of_a_column(shop):
|
||||
prober = make_prober({shop.name: shop}, clock=lambda: CLOCK)
|
||||
probe = await prober(shop.name, "orders", "order_date")
|
||||
assert probe == {"source": shop.name, "table": "orders", "column": "order_date", "min": "2026-01-01",
|
||||
"max": LAST_DAY.isoformat(), "basis": "live", "at": CLOCK}
|
||||
assert prober.dialect(shop.name) == "sqlite" and prober.dialect("nope") is None
|
||||
assert await prober("nope", "orders", "order_date") is None
|
||||
|
||||
|
||||
async def test_masked_columns_are_never_probed(shop):
|
||||
prober = make_prober({shop.name: shop})
|
||||
assert await prober(shop.name, "customers", "signup_date") is None # 数据源设的遮罩列
|
||||
explicit = make_prober({shop.name: shop}, masked={shop.name: ["ORDER_DATE"]})
|
||||
assert await explicit(shop.name, "orders", "order_date") is None
|
||||
|
||||
|
||||
async def test_a_failed_probe_falls_back_to_profile_facts(shop):
|
||||
async def facts(table):
|
||||
return {"at": "2026-10-20T02:00:00", "date_ranges": [{"column": "paid_on", "min": "2026-02-01",
|
||||
"max": "2026-10-19"}]}
|
||||
|
||||
prober = make_prober({shop.name: shop}, facts={shop.name: facts}, clock=lambda: CLOCK)
|
||||
probe = await prober(shop.name, "payments", "paid_on") # 没有这张表:现查不成
|
||||
assert probe == {"source": shop.name, "table": "payments", "column": "paid_on", "min": "2026-02-01",
|
||||
"max": "2026-10-19", "basis": "profile", "at": "2026-10-20"}
|
||||
|
||||
|
||||
async def test_the_budget_stops_further_probes(shop):
|
||||
prober = make_prober({shop.name: shop}, budget=0)
|
||||
await prober(shop.name, "orders", "order_date")
|
||||
assert await prober(shop.name, "orders", "order_date") is None
|
||||
|
||||
|
||||
async def test_bounds_are_computed_in_the_database(shop):
|
||||
prober = make_prober({shop.name: shop}, clock=lambda: CLOCK)
|
||||
probe = await prober.bounds(shop.name, "DATE('now', '-7 days')")
|
||||
assert probe["expr"] == "DATE('now', '-7 days')" and probe["basis"] == "live" and probe["at"] == CLOCK
|
||||
assert date.fromisoformat(probe["value"]) == UTC_TODAY - timedelta(days=7)
|
||||
assert await prober.bounds(shop.name, "no_such_function(1)") is None
|
||||
|
||||
|
||||
def test_the_book_registers_first_and_records_only_what_was_used():
|
||||
book = Book({})
|
||||
assert book.range("shop", "orders", "order_date") is None and book.bound("shop", "DATE('now')") is None
|
||||
assert book.missing == [("range", "shop", "orders", "order_date"), ("bound", "shop", "DATE('now')")]
|
||||
found = {"source": "shop", "table": "orders", "column": "order_date", "min": "2026-01-01", "max": "2026-10-18",
|
||||
"basis": "live", "at": CLOCK}
|
||||
filled = Book({("range", "shop", "orders", "order_date"): found, ("bound", "shop", "DATE('now')"): None})
|
||||
assert filled.range("shop", "ORDERS", "Order_Date") is found # 表名、列名不分大小写
|
||||
filled.bound("shop", "DATE('now')")
|
||||
assert filled.missing == [] and filled.used == [found]
|
||||
replay = Book([found]) # 复算:记下的就是全部
|
||||
assert replay.bound("shop", "DATE('now', '-1 day')") is None and replay.missing == []
|
||||
|
||||
|
||||
async def test_a_full_comparability_check_against_the_database_records_its_probes(shop):
|
||||
sql = ("SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', '{a}') "
|
||||
"AND order_date < date('now', '{b}')")
|
||||
base = snap("Q1", sql.format(a="-14 days", b="-7 days"), ["gmv"], [[700]], source=shop.name)
|
||||
target = snap("Q2", sql.format(a="-7 days", b="0 days"), ["gmv"], [[300]], source=shop.name)
|
||||
prober = make_prober({shop.name: shop}, clock=lambda: CLOCK)
|
||||
result = await analysis.run({"op": "comparable", "params": {"base_query": "Q1", "target_query": "Q2",
|
||||
"measure": "gmv"}},
|
||||
{"Q1": base, "Q2": target}.get, settings=effective_settings(None),
|
||||
context={"today": UTC_TODAY, "probe": prober,
|
||||
"catalogs": {shop.name: CATALOGS["shop"]}})
|
||||
exprs = [p["expr"] for p in result["probes"] if "expr" in p]
|
||||
assert exprs == ["DATE('now', '-14 days')", "DATE('now', '-7 days')", "DATE('now', '0 days')"]
|
||||
ranges = [p for p in result["probes"] if "table" in p]
|
||||
assert [(p["table"], p["column"], p["max"], p["basis"]) for p in ranges] == \
|
||||
[("orders", "order_date", LAST_DAY.isoformat(), "live")]
|
||||
assert result["values"]["base_days"] == "7" and result["values"]["target_days"] == "7"
|
||||
assert result["values"]["target_end"] == LAST_DAY.isoformat()
|
||||
assert all(i["dialect"] == "sqlite" for i in result["inputs"])
|
||||
assert result["verdict"]["code"] == "comparable" and result["phrase"] == "两期可比"
|
||||
assert same(result, recomputed(result, [base, target]))
|
||||
@@ -0,0 +1,152 @@
|
||||
"""analysis.run / recompute(app/analysis/__init__.py):结论的形状严格按契约、同输入两次 run 逐字节相同、复算和运行逐字段相同且
|
||||
只用记下的 probes(不现查、不读目录)、调用方写的系统字段不收、复算取不回输入时的说法。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from app import analysis
|
||||
from app.analysis import contract as c
|
||||
from app.analysis.contract import CallError, effective_settings, same
|
||||
from tests.analysis_samples import CATALOGS, analyze, call_error, fixed_prober, recomputed, snap
|
||||
|
||||
WEEKLY_SQL = "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week"
|
||||
WEEKLY = snap("Q2", WEEKLY_SQL, ["region", "week", "gmv"],
|
||||
[["华东", "2026-W41", 820], ["华东", "2026-W42", 700], ["华北", "2026-W41", 600],
|
||||
["华北", "2026-W42", 560], ["华南", "2026-W41", 700], ["华南", "2026-W42", 666]])
|
||||
SERIES = snap("Q1", "SELECT DATE(order_date, 'weekday 0', '-6 days') AS week, SUM(amount) AS gmv FROM orders GROUP BY 1",
|
||||
["week", "gmv"], [[(date(2026, 6, 1) + timedelta(days=7 * i)).isoformat(), 100 + (i % 4) * 3 + i]
|
||||
for i in range(20)])
|
||||
RATES = snap("Q3", "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region",
|
||||
["region", "refunds", "orders"], [["西南", 80, 1000], ["华东", 40, 1000], ["华北", 41, 1000]])
|
||||
BASE = snap("Q4", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' AND "
|
||||
"order_date < '2026-10-12' GROUP BY region", ["region", "gmv"], [["华东", 820], ["华北", 600]])
|
||||
TARGET = snap("Q5", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-12' AND "
|
||||
"order_date < '2026-10-19' GROUP BY region", ["region", "gmv"], [["华东", 700], ["华北", 590]])
|
||||
SNAPS = [WEEKLY, SERIES, RATES, BASE, TARGET]
|
||||
PROBE = {("orders", "order_date"): ("2025-01-01", "2026-10-20")}
|
||||
|
||||
CALLS = [
|
||||
("decompose", "additive", {"query": "Q2", "by": "region", "period": "week", "base": "2026-W41",
|
||||
"target": "2026-W42", "measure": "gmv", "label": "销售额"}),
|
||||
("decompose", "additive", {"base_query": "Q4", "target_query": "Q5", "by": "region", "measure": "gmv"}),
|
||||
("baseline", None, {"query": "Q1", "period": "week", "measure": "gmv"}),
|
||||
("trend", None, {"query": "Q1", "period": "week", "measure": "gmv"}),
|
||||
("differ", "rate", {"query": "Q3", "by": "region", "numerator": "refunds", "denominator": "orders"}),
|
||||
("comparable", None, {"base_query": "Q4", "target_query": "Q5", "measure": "gmv"}),
|
||||
]
|
||||
|
||||
|
||||
async def _run(op, mode, params, *, probe=None, **kw):
|
||||
return await analyze(op, mode, snaps=SNAPS, probe=probe or fixed_prober(PROBE), check=False, **params, **kw)
|
||||
|
||||
|
||||
def _assert_shape(result):
|
||||
"""结论的字段、取值严格按契约。"""
|
||||
assert set(result) == set(c.AnalysisResult.__annotations__)
|
||||
assert result["op"] in c.OPS and result["method"] == c.METHODS[(result["op"], result["mode"])]
|
||||
assert result["status"] in ("ok", "refused")
|
||||
if result["status"] == "refused":
|
||||
assert result["reason_code"] in c.REASONS and result["reason"] and result["verdict"] is None
|
||||
else:
|
||||
assert result["reason_code"] is None and result["reason"] is None
|
||||
assert result["verdict"]["code"] in c.VERDICTS[result["op"]]
|
||||
assert set(result["verdict"]) == {"code", "subjects", "direction"}
|
||||
assert all(isinstance(s, list) and all(isinstance(x, str) for x in s) for s in result["verdict"]["subjects"])
|
||||
assert result["verdict"]["direction"] in (None, "up", "down")
|
||||
table = result["table"]
|
||||
assert set(table) == {"columns", "labels", "rows"} and set(table["labels"]) == set(table["columns"])
|
||||
assert all(len(r) == len(table["columns"]) for r in table["rows"])
|
||||
assert all(v is None or isinstance(v, str) for v in result["values"].values())
|
||||
assert all(set(cv) == {"code", "text"} and cv["code"] in c.CAVEATS for cv in result["caveats"])
|
||||
assert set(result["settings"]) <= set(c.DEFAULTS)
|
||||
assert all(i.get("artifact") for i in result["inputs"])
|
||||
assert result["criterion"] and result["phrase"]
|
||||
json.dumps(result, ensure_ascii=False) # 写得进 JSON
|
||||
|
||||
|
||||
@pytest.mark.parametrize("op,mode,params", CALLS)
|
||||
async def test_every_result_has_the_contract_shape_and_recomputes_the_same(op, mode, params):
|
||||
result = await _run(op, mode, params)
|
||||
_assert_shape(result)
|
||||
again = recomputed(result, SNAPS)
|
||||
assert same(result, again)
|
||||
assert json.dumps(again, sort_keys=True, ensure_ascii=False) == json.dumps(result, sort_keys=True, ensure_ascii=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("op,mode,params", CALLS)
|
||||
async def test_the_same_inputs_give_byte_identical_results(op, mode, params):
|
||||
one = await _run(op, mode, params)
|
||||
two = await _run(op, mode, params)
|
||||
assert json.dumps(one, sort_keys=True, ensure_ascii=False) == json.dumps(two, sort_keys=True, ensure_ascii=False)
|
||||
|
||||
|
||||
async def test_recompute_uses_the_recorded_probes_and_facts_not_live_ones():
|
||||
result = await _run("comparable", None, {"base_query": "Q4", "target_query": "Q5", "measure": "gmv"})
|
||||
assert result["verdict"]["code"] == "comparable" and len(result["probes"]) == 1
|
||||
# 记下的范围换一个(表里数据只到 10-16):复算按记下的算,结论跟着变——它没有现查
|
||||
edited = {**result, "probes": [{**result["probes"][0], "max": "2026-10-16"}]}
|
||||
assert recomputed(edited, SNAPS)["verdict"]["code"] == "incomparable"
|
||||
# 目录事后改了也不影响复算:目录事实记在 params 里
|
||||
decompose = await _run(*CALLS[0])
|
||||
assert decompose["params"]["catalog"]["labels"]["region"] == "地区"
|
||||
CATALOGS["shop"]["orders"]["columns"]["region"]["label"]["value"] = "大区"
|
||||
try:
|
||||
assert same(decompose, recomputed(decompose, SNAPS))
|
||||
finally:
|
||||
CATALOGS["shop"]["orders"]["columns"]["region"]["label"]["value"] = "地区"
|
||||
|
||||
|
||||
async def test_system_params_written_by_the_caller_are_ignored():
|
||||
params = {"query": "Q1", "period": "week", "measure": "gmv", "today": "2020-01-01",
|
||||
"catalog": {"labels": {"gmv": "假的叫法"}}, "depth": 9}
|
||||
result = await analysis.run({"op": "baseline", "params": params}, {s["ref"]: s for s in SNAPS}.get,
|
||||
settings=effective_settings(None),
|
||||
context={"today": date(2026, 10, 21), "catalogs": CATALOGS})
|
||||
assert result["params"]["today"] == "2026-10-21"
|
||||
assert "假的叫法" not in result["phrase"] and "depth" not in result["params"]
|
||||
|
||||
|
||||
async def test_analysis_card_overrides_reach_the_criterion_and_settings():
|
||||
params = CALLS[0][2]
|
||||
result = await _run("decompose", "additive", params, settings={"main_share": 0.6})
|
||||
assert result["settings"]["main_share"] == 0.6 and "占总变化的 60% 以上" in result["criterion"]
|
||||
|
||||
|
||||
async def test_call_errors_for_unknown_ops_and_modes():
|
||||
with pytest.raises(CallError) as e:
|
||||
await analysis.run({"op": "forecast", "params": {}}, {}.get, settings=effective_settings(None), context={})
|
||||
assert e.value.code == "unknown_op" and "变化拆解" in e.value.message
|
||||
with pytest.raises(CallError) as e:
|
||||
await analysis.run({"op": "decompose", "mode": "share", "params": {}}, {}.get,
|
||||
settings=effective_settings(None), context={})
|
||||
assert e.value.code == "unknown_mode" and "additive / ratio / factor" in e.value.message
|
||||
missing = await call_error("baseline", snaps=SNAPS, query="Q1", period="week")
|
||||
assert missing.code == "missing_param" and missing.message == "没写度量列"
|
||||
|
||||
|
||||
async def test_recompute_says_when_an_input_cannot_be_loaded():
|
||||
result = await _run(*CALLS[0])
|
||||
with pytest.raises(CallError) as e:
|
||||
analysis.recompute(result, lambda artifact: None)
|
||||
assert e.value.code == "input_missing" and e.value.message == "复算取不回 Q2 的查询快照"
|
||||
|
||||
|
||||
async def test_the_hypothesis_is_recorded_but_never_rendered():
|
||||
result = await _run(*CALLS[0], hypothesis="可能是华东促销结束 30% 的影响")
|
||||
assert result["hypothesis"] == "可能是华东促销结束 30% 的影响"
|
||||
assert "促销" not in result["phrase"] and "30" not in result["phrase"]
|
||||
assert same(result, {**recomputed(result, SNAPS), "hypothesis": None})
|
||||
|
||||
|
||||
async def test_ledger_entries_name_the_real_input_snapshots():
|
||||
result = await _run(*CALLS[1])
|
||||
entry = c.ledger_entry(result, node_id="agent", exec_no=1, artifact="f" * 64, call_id="call-1")
|
||||
assert entry["inputs"] == [BASE["artifact"], TARGET["artifact"]] and entry["op"] == "decompose"
|
||||
|
||||
|
||||
def test_the_call_limit_error_is_spoken_with_the_limit():
|
||||
error = analysis.exhausted({"max_calls": 5})
|
||||
assert error.code == "max_calls" and error.message == "这次运行的分析次数已经用完(最多 5 次):用已有的分析结论写,不再新做分析"
|
||||
@@ -0,0 +1,210 @@
|
||||
"""波动基线、趋势(app/analysis/series.py,细案 01 §3、§4):常规 / 偏离 / 明显偏离、σ̂ = 0、历史不够、本期不完整、缺期、日数据
|
||||
lag 7、新高新低;上升 / 下降 / 无趋势、连续期数、缺期。每个用例都顺带核对复算和运行逐字段相同。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, timedelta
|
||||
from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
|
||||
from app.analysis.series import MAD_SCALE, run_length
|
||||
from tests.analysis_samples import analyze, call_error, fixed_prober, snap
|
||||
|
||||
SQL = "SELECT DATE(order_date, 'weekday 0', '-6 days') AS week, COUNT(*) AS orders FROM orders GROUP BY 1"
|
||||
#: 每周一的日期:最后一周是 2026-10-12(到 10-18,运行当天 10-21 之前已经过完)
|
||||
LAST_MONDAY = date(2026, 10, 12)
|
||||
|
||||
|
||||
def weeks(values, *, last=LAST_MONDAY):
|
||||
start = last - timedelta(days=7 * (len(values) - 1))
|
||||
return [[(start + timedelta(days=7 * i)).isoformat(), v] for i, v in enumerate(values)]
|
||||
|
||||
|
||||
def series(rows, sql=SQL, cols=("week", "orders")):
|
||||
return snap("Q1", sql, list(cols), rows, types={cols[0]: "text", cols[1]: "number"})
|
||||
|
||||
|
||||
async def baseline(rows, **params):
|
||||
params = {"query": "Q1", "period": "week", "measure": "orders", "label": "订单数", **params}
|
||||
return await analyze("baseline", snaps=[series(rows)], **params)
|
||||
|
||||
|
||||
async def trend(rows, **params):
|
||||
params = {"query": "Q1", "period": "week", "measure": "orders", "label": "线上销售额", **params}
|
||||
return await analyze("trend", snaps=[series(rows)], **params)
|
||||
|
||||
|
||||
NOISE = [100, 102, 98, 101, 99, 103, 100, 97, 102, 101, 99, 100, 98, 103, 101, 100, 99, 102, 100]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 波动基线
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_level_basis_with_hand_computed_median_and_mad():
|
||||
rows = [[i + 1, v] for i, v in enumerate([10, 12, 11, 13, 9, 10, 12, 11, 16])]
|
||||
result = await baseline(rows, basis="level")
|
||||
# 历史中位数 11,MAD 1,σ̂ = 1.4826,z = (16 − 11) / 1.4826
|
||||
values = result["values"]
|
||||
assert values["median"] == "11" and values["n_history"] == "8" and values["rank"] == "1"
|
||||
assert Fraction(values["z"]) == pytest.approx(Fraction(5) / MAD_SCALE)
|
||||
assert values["band_lo"] == "8.0348" and values["band_hi"] == "13.9652"
|
||||
assert result["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "up"}
|
||||
assert result["phrase"].startswith("订单数本期为 16,明显超出过去 8 期的常规波动范围(8.03 至 13.97)")
|
||||
assert any(c["code"] == "record_high" for c in result["caveats"])
|
||||
assert result["params"]["grain"] == "seq" and result["params"]["lag"] == 1
|
||||
assert result["settings"] == {"k_warn": 2.0, "k_alert": 3.0, "min_history": 8, "max_history": 52}
|
||||
assert result["table"]["rows"][-1][-1] == "本期"
|
||||
|
||||
|
||||
async def test_change_basis_within_outside_and_far_outside():
|
||||
probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")})
|
||||
within = await analyze("baseline", snaps=[series(weeks(NOISE + [101]))], probe=probe, query="Q1", period="week",
|
||||
measure="orders", label="订单数")
|
||||
assert within["verdict"]["code"] == "within_band"
|
||||
assert within["phrase"].startswith("订单数本期变化 +1.0%,在过去 18 周的常规波动范围内(")
|
||||
assert within["params"]["today"] == "2026-10-21" and within["params"]["grain"] == "week"
|
||||
assert within["probes"][0]["table"] == "orders" and within["probes"][0]["column"] == "order_date"
|
||||
assert not any(c["code"] == "comparability" for c in within["caveats"])
|
||||
outside = await analyze("baseline", snaps=[series(weeks(NOISE + [107]))], probe=probe, query="Q1",
|
||||
period="week", measure="orders")
|
||||
assert outside["verdict"]["code"] == "outside_band" and "超出过去 18 周" in outside["phrase"]
|
||||
far = await analyze("baseline", snaps=[series(weeks(NOISE + [70]))], probe=probe, query="Q1", period="week",
|
||||
measure="orders", label="订单数")
|
||||
assert far["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "down"}
|
||||
assert far["phrase"].startswith("订单数本期变化 -30.0%,明显超出过去 18 周的常规波动范围")
|
||||
assert any(c["code"] == "record_low" and c["text"] == "本期的订单数是过去 18 周里最低的" for c in far["caveats"])
|
||||
|
||||
|
||||
async def test_without_a_table_range_completeness_is_said_to_be_unverified_but_not_in_the_phrase():
|
||||
result = await baseline(weeks(NOISE + [101]))
|
||||
note = next(c for c in result["caveats"] if c["code"] == "comparability")
|
||||
assert note["text"] == "没能核对本期的数据是否到齐(查不到表里这一列的日期范围)"
|
||||
assert note["text"] not in result["phrase"]
|
||||
|
||||
|
||||
async def test_flat_history_has_no_spread():
|
||||
rows = [[i + 1, 50] for i in range(10)]
|
||||
same = await baseline(rows, basis="level")
|
||||
assert same["verdict"]["code"] == "within_band" and same["values"]["z"] is None
|
||||
assert same["phrase"] == "订单数本期为 50,和过去 9 期一样(每期都是 50)"
|
||||
jump = await baseline(rows[:-1] + [[10, 51]], basis="level")
|
||||
assert jump["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "up"}
|
||||
assert jump["phrase"].startswith("订单数本期为 51,明显超出过去 9 期的常规波动范围(每期都是 50)")
|
||||
|
||||
|
||||
async def test_short_history_partial_target_gaps_and_zero_bases_are_refused():
|
||||
short = await baseline(weeks([100, 101, 99, 102, 98, 100]))
|
||||
# 6 周:第一周没有上一周可比,本期不算历史,历史的变化只有 4 个
|
||||
assert short["reason_code"] == "history_short" and short["reason"] == "只有 4 期历史,至少要 8 期"
|
||||
assert short["phrase"] == "波动基线没有做成:只有 4 期历史,至少要 8 期"
|
||||
# 最后一周是 10-19 到 10-25:运行当天 10-21 还没过完
|
||||
partial = await baseline(weeks(NOISE + [60], last=date(2026, 10, 19)))
|
||||
assert partial["reason_code"] == "partial_target"
|
||||
assert partial["reason"] == "本期数据只到 2026-10-20,还差 5 天,和完整的周比没有意义"
|
||||
# 周过完了,但表里的数据只到 10-17
|
||||
probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-17")})
|
||||
late = await analyze("baseline", snaps=[series(weeks(NOISE + [60]))], probe=probe, query="Q1", period="week",
|
||||
measure="orders")
|
||||
assert late["reason"] == "本期数据只到 2026-10-17,还差 1 天,和完整的周比没有意义"
|
||||
gap_rows = weeks(NOISE + [100, 100])
|
||||
del gap_rows[-2] # 本期的上一周没有
|
||||
gap = await baseline(gap_rows)
|
||||
assert gap["reason_code"] == "target_gap"
|
||||
zero = await baseline(weeks(NOISE + [0, 10]))
|
||||
assert zero["reason_code"] == "zero_base"
|
||||
unreadable = await baseline([["上周", 1], ["本周", 2]])
|
||||
assert unreadable["reason_code"] == "period_unreadable" and "上周" in unreadable["reason"]
|
||||
|
||||
|
||||
async def test_daily_series_compare_with_the_same_weekday_last_week():
|
||||
start = date(2026, 7, 1)
|
||||
rows = [[(start + timedelta(days=i)).isoformat(), 100 + (i % 7) * 10] for i in range(100)]
|
||||
result = await analyze("baseline", snaps=[series(rows, cols=("day", "orders"))], query="Q1", period="day",
|
||||
measure="orders", label="订单数", check=True)
|
||||
assert result["params"]["grain"] == "day" and result["params"]["lag"] == 7
|
||||
# 星期效应:和上周同一天比,每天的变化都是 0
|
||||
assert result["values"]["median"] == "0" and result["verdict"]["code"] == "within_band"
|
||||
assert result["phrase"].startswith("订单数比上周同一天变化 0.0%,和过去 52 天一样")
|
||||
yoy = await baseline(weeks(NOISE * 3), lag=52) # 57 周:能和 52 周前比的历史只有 4 期
|
||||
assert yoy["reason_code"] == "history_short" and yoy["reason"] == "只有 4 期历史,至少要 8 期"
|
||||
assert (await baseline(weeks(NOISE * 4), lag=52))["status"] == "ok"
|
||||
assert (await call_error("baseline", snaps=[series(weeks(NOISE))], query="Q1", period="week", measure="orders",
|
||||
lag=0)).code == "bad_lag"
|
||||
|
||||
|
||||
async def test_duplicate_periods_and_given_targets():
|
||||
rows = weeks(NOISE + [101])
|
||||
error = await call_error("baseline", snaps=[series(rows + [rows[-1]])], query="Q1", period="week",
|
||||
measure="orders")
|
||||
assert error.code == "duplicate_rows" and "在 SQL 里按 week 汇总" in error.message
|
||||
earlier = await baseline(rows, target=rows[-3][0])
|
||||
assert earlier["params"]["target"] == rows[-3][0] and earlier["values"]["n_history"] == "16"
|
||||
missing = await call_error("baseline", snaps=[series(rows)], query="Q1", period="week", measure="orders",
|
||||
target="2020-01-06")
|
||||
assert missing.code == "period_not_found"
|
||||
|
||||
|
||||
async def test_iso_week_labels_and_months():
|
||||
labels = [[f"2026-W{w:02d}", v] for w, v in zip(range(22, 42), NOISE + [101])]
|
||||
result = await baseline(labels)
|
||||
assert result["params"]["grain"] == "week" and result["verdict"]["code"] == "within_band"
|
||||
months = [[f"2025-{m:02d}", 100 + m] for m in range(1, 13)] + [[f"2026-{m:02d}", 112 + m] for m in range(1, 10)]
|
||||
monthly = await baseline(months)
|
||||
assert monthly["params"]["grain"] == "month" and "过去 19 个月" in monthly["phrase"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 趋势
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_rising_and_falling_trends_with_run_length():
|
||||
rising = await trend(weeks([100 + 3 * i + (i % 3) for i in range(12)]))
|
||||
assert rising["verdict"] == {"code": "rising", "subjects": [], "direction": "up"}
|
||||
assert rising["values"]["n"] == "12" and rising["values"]["run_length"] == "11"
|
||||
assert rising["phrase"].startswith("过去 12 周线上销售额呈上升趋势(每周约 +")
|
||||
assert ",已连续 11 周上升" in rising["phrase"]
|
||||
assert rising["settings"] == {"trend_min_points": 6, "trend_alpha": 0.05, "max_history": 52}
|
||||
falling = await trend(weeks([200, 196, 197, 190, 185, 186, 180, 181, 175, 170]))
|
||||
assert falling["verdict"]["code"] == "falling"
|
||||
assert falling["values"]["run_length"] == "2" and "已连续" not in falling["phrase"]
|
||||
|
||||
|
||||
async def test_no_trend_and_exact_small_sample_p():
|
||||
flat = await trend(weeks([100, 104, 98, 103, 97, 101, 99, 102]))
|
||||
assert flat["verdict"] == {"code": "no_trend", "subjects": [], "direction": None}
|
||||
assert flat["phrase"] == "过去 8 周线上销售额没有明显的上升或下降趋势"
|
||||
six = await trend(weeks([1, 2, 3, 4, 5, 6]))
|
||||
# 没有并列、n ≤ 10:精确分布,单调的 6 期双侧 p = 2 / 6!
|
||||
assert Fraction(six["values"]["p"]) == pytest.approx(Fraction(2, 720))
|
||||
assert six["verdict"]["code"] == "rising" and six["values"]["slope"] == "1"
|
||||
|
||||
|
||||
def test_run_length_counts_consecutive_changes_from_the_end():
|
||||
assert run_length([Fraction(v) for v in (1, 2, 3, 2, 3, 4, 5)]) == (3, "up")
|
||||
assert run_length([Fraction(v) for v in (5, 4, 3)]) == (2, "down")
|
||||
assert run_length([Fraction(v) for v in (1, 2, 2)]) == (0, None)
|
||||
|
||||
|
||||
async def test_trend_needs_enough_consecutive_points():
|
||||
rows = weeks([100 + i for i in range(10)])
|
||||
del rows[-4] # 最后连着的只有 3 期
|
||||
gap = await trend(rows)
|
||||
assert gap["reason_code"] == "series_gap" and gap["reason"] == "最后连着的只有 3 期,再往前缺期(至少要连着的 6 期)"
|
||||
short = await trend(weeks([1, 2, 3, 4]))
|
||||
assert short["reason_code"] == "history_short" and short["reason"] == "只有 4 期,至少要 6 期"
|
||||
partial = await trend(weeks([100 + i for i in range(10)], last=date(2026, 10, 19)))
|
||||
assert partial["reason_code"] == "partial_target"
|
||||
|
||||
|
||||
async def test_daily_trends_warn_about_weekday_effects():
|
||||
start = date(2026, 8, 1)
|
||||
rows = [[(start + timedelta(days=i)).isoformat(), 100 + i] for i in range(30)]
|
||||
result = await analyze("trend", snaps=[series(rows, cols=("day", "orders"))], query="Q1", period="day",
|
||||
measure="orders")
|
||||
assert result["verdict"]["code"] == "rising"
|
||||
assert any(c["code"] == "weekday_effect" and c["text"] == "日数据有星期效应,看趋势建议按周汇总"
|
||||
for c in result["caveats"])
|
||||
assert "日数据有星期效应" in result["phrase"]
|
||||
@@ -0,0 +1,184 @@
|
||||
"""分析用的统计函数(app/analysis/stats.py):用公认的表值钉住,外加边界——并列、n 很小、p 接近 0 / 1、t 自由度很大趋近正态。
|
||||
Wilson 从 eval/stats 搬过来,评测引用的是同一个函数、结果和以前一样。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import math
|
||||
import random
|
||||
from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
|
||||
from app.analysis import stats
|
||||
from app.eval import stats as eval_stats
|
||||
|
||||
|
||||
def test_normal_quantiles_and_distribution_match_the_tables():
|
||||
assert stats.normal_ppf(0.975) == pytest.approx(1.959964, abs=1e-6)
|
||||
assert stats.normal_ppf(0.975) == pytest.approx(stats.Z95, rel=1e-15)
|
||||
assert stats.normal_ppf(0.5) == 0.0
|
||||
assert stats.normal_ppf(0.95) == pytest.approx(1.644854, abs=1e-6)
|
||||
assert stats.normal_ppf(0.995) == pytest.approx(2.575829, abs=1e-6)
|
||||
assert stats.normal_ppf(1e-10) == pytest.approx(-6.361341, abs=1e-6) # 很靠近 0 的尾巴
|
||||
assert stats.normal_ppf(0) == -math.inf and stats.normal_ppf(1) == math.inf
|
||||
for p in (1e-12, 0.001, 0.02425, 0.3, 0.7, 0.97575, 0.999, 1 - 1e-9):
|
||||
assert stats.normal_cdf(stats.normal_ppf(p)) == pytest.approx(p, rel=1e-9)
|
||||
assert stats.normal_cdf(1.959963984540054) == pytest.approx(0.975, abs=1e-15)
|
||||
assert stats.normal_two_sided(1.959963984540054) == pytest.approx(0.05, abs=1e-15)
|
||||
|
||||
|
||||
def test_t_quantiles_match_the_tables():
|
||||
# 双侧 0.05 的临界值(t 分布表)
|
||||
table = {1: 12.706205, 2: 4.302653, 5: 2.570582, 10: 2.228139, 30: 2.042272, 100: 1.983972}
|
||||
for df, value in table.items():
|
||||
assert stats.t_ppf(0.975, df) == pytest.approx(value, abs=1e-6), df
|
||||
assert stats.t_two_sided(2.228139, 10) == pytest.approx(0.05, abs=1e-6)
|
||||
# 自由度 2 有闭式解:F(t) = 1/2 + t / (2√(2 + t²))
|
||||
for t in (0.3, 1.5, 4.0, 25.0):
|
||||
assert stats.t_cdf(t, 2) == pytest.approx(0.5 + t / (2 * math.sqrt(2 + t * t)), rel=1e-12)
|
||||
assert stats.t_cdf(0.0, 7) == 0.5 and stats.t_two_sided(0.0, 7) == 1.0
|
||||
assert stats.t_cdf(-2.0, 10) == pytest.approx(1 - stats.t_cdf(2.0, 10), rel=1e-12)
|
||||
|
||||
|
||||
def test_t_with_a_huge_df_approaches_the_normal():
|
||||
assert stats.t_ppf(0.975, 1e6) == pytest.approx(stats.Z95, abs=1e-5)
|
||||
assert stats.t_ppf(0.975, 1e9) == pytest.approx(stats.Z95, abs=1e-9)
|
||||
assert stats.t_two_sided(1.96, 5e6) == pytest.approx(stats.normal_two_sided(1.96), abs=1e-6)
|
||||
# 非整数自由度(Welch)也行
|
||||
assert stats.t_ppf(0.975, 41.784) == pytest.approx(2.018, abs=1e-3)
|
||||
|
||||
|
||||
def test_regularized_incomplete_beta():
|
||||
assert stats.betainc(2, 3, 0.4) == pytest.approx(0.5248, abs=1e-12) # 多项式,手算
|
||||
assert stats.betainc(0.5, 0.5, 0.5) == pytest.approx(0.5, abs=1e-12)
|
||||
assert stats.betainc(1, 1, 0.3) == pytest.approx(0.3, abs=1e-12)
|
||||
assert stats.betainc(3, 2, 0.0) == 0.0 and stats.betainc(3, 2, 1.0) == 1.0
|
||||
|
||||
|
||||
def test_wilson_moved_here_is_the_same_function_the_eval_uses():
|
||||
assert eval_stats.wilson is stats.wilson
|
||||
assert eval_stats.Z95 == stats.Z95
|
||||
assert stats.wilson(8, 10) == (0.4902, 0.9433)
|
||||
assert stats.wilson(0, 0) is None
|
||||
assert stats.wilson(0, 5)[0] == 0.0 and stats.wilson(5, 5)[1] == 1.0
|
||||
low, high = stats.wilson(8, 10, places=None)
|
||||
assert round(low, 4) == 0.4902 and round(high, 4) == 0.9433 and low != 0.4902
|
||||
|
||||
|
||||
def test_newcombe_matches_the_published_example():
|
||||
# Newcombe(1998)的例子:56/70 对 48/80,混合 score 法(方法 10)0.0524 到 0.3339
|
||||
low, high = stats.newcombe(56, 70, 48, 80)
|
||||
assert low == pytest.approx(0.0524, abs=1e-4) and high == pytest.approx(0.3339, abs=1e-4)
|
||||
assert stats.newcombe(1, 0, 2, 5) is None
|
||||
|
||||
|
||||
def test_pooled_z_and_min_expected():
|
||||
z, p = stats.pooled_z(56, 70, 48, 80)
|
||||
assert z == pytest.approx(2.6502, abs=1e-4) and p == pytest.approx(0.00805, abs=1e-5)
|
||||
assert stats.pooled_z(0, 10, 0, 20) == (0.0, 1.0) # 两边全不是:没有差异可言
|
||||
assert stats.pooled_z(3, 0, 1, 5) is None
|
||||
assert stats.min_expected(2, 40, 5, 230) == pytest.approx(7 * 40 / 270)
|
||||
|
||||
|
||||
def test_fisher_exact_two_sided():
|
||||
# Fisher 的品茶实验
|
||||
assert stats.fisher_exact(3, 1, 1, 3) == pytest.approx(0.4857142857, abs=1e-9)
|
||||
assert stats.fisher_exact(1, 9, 11, 3) == pytest.approx(0.002759456, abs=1e-8)
|
||||
assert stats.fisher_exact(0, 10, 10, 0) == pytest.approx(2 / math.comb(20, 10), rel=1e-9) # p 接近 0
|
||||
assert stats.fisher_exact(5, 5, 5, 5) == pytest.approx(1.0) # p 接近 1
|
||||
assert stats.fisher_exact(0, 0, 0, 0) == 1.0
|
||||
|
||||
|
||||
def test_welch_t_test():
|
||||
test = stats.welch(10, 4, 20, 8, 9, 25)
|
||||
# t = 2 / √(4/20 + 9/25) = 2.6726;自由度按 Welch–Satterthwaite
|
||||
assert test.t == pytest.approx(2.672612, abs=1e-6)
|
||||
assert test.df == pytest.approx(0.56 ** 2 / (0.2 ** 2 / 19 + 0.36 ** 2 / 24), rel=1e-12)
|
||||
assert test.p == pytest.approx(stats.t_two_sided(test.t, test.df))
|
||||
assert stats.welch(1, 1, 1, 2, 1, 5) is None # 一组只有一个样本:方差算不了
|
||||
assert stats.welch(5, 0, 3, 5, 0, 4).p == 1.0 # 没有方差、均值一样
|
||||
assert stats.welch(5, 0, 3, 6, 0, 4).p == 0.0
|
||||
low, high = stats.welch_interval(10, 4, 20, 8, 9, 25)
|
||||
assert low < 2 < high and (low + high) / 2 == pytest.approx(2)
|
||||
|
||||
|
||||
def test_holm_step_down():
|
||||
assert stats.holm([0.01, 0.04, 0.03, 0.005]) == pytest.approx([0.03, 0.06, 0.06, 0.02])
|
||||
assert stats.holm([0.01, None, 0.5]) == [0.02, None, 0.5]
|
||||
assert stats.holm([0.6, 0.9]) == [1.0, 1.0] # 不超过 1
|
||||
assert stats.holm([]) == []
|
||||
|
||||
|
||||
def _exact_two_sided(values: list[int]) -> Fraction:
|
||||
"""暴力枚举全部排列,算 Mann–Kendall 的精确双侧 p(对照用)。"""
|
||||
def s_of(seq: tuple[int, ...]) -> int:
|
||||
return sum((b > a) - (b < a) for i, a in enumerate(seq) for b in seq[i + 1:])
|
||||
|
||||
observed = abs(s_of(tuple(values)))
|
||||
perms = list(itertools.permutations(values))
|
||||
return Fraction(sum(1 for p in perms if abs(s_of(p)) >= observed), len(perms))
|
||||
|
||||
|
||||
def test_mann_kendall_small_samples_use_the_exact_distribution():
|
||||
mk = stats.mann_kendall([1, 2, 3, 4])
|
||||
assert mk.exact and mk.s == 6 and mk.p == Fraction(1, 12) # 4! 里只有两头两个排列
|
||||
assert stats.mann_kendall([1, 2, 3, 4, 5]).p == Fraction(1, 60)
|
||||
for values in ([3, 1, 4, 2, 6, 5], [1, 3, 2, 5, 4, 7, 6], [2, 1, 4, 3, 6, 5, 8]):
|
||||
assert stats.mann_kendall(values).p == _exact_two_sided(values)
|
||||
# n = 10 的单调序列:2 / 10!
|
||||
assert stats.mann_kendall(list(range(10))).p == Fraction(2, math.factorial(10))
|
||||
|
||||
|
||||
def test_mann_kendall_with_ties_or_more_points_uses_the_corrected_normal_approximation():
|
||||
values = [1, 2, 2, 3, 4, 5, 5, 6, 7, 8, 9, 10]
|
||||
mk = stats.mann_kendall(values)
|
||||
n = len(values)
|
||||
var = (n * (n - 1) * (2 * n + 5) - 2 * (2 * 1 * 9)) / 18 # 两组各两个并列
|
||||
assert not mk.exact and mk.var == pytest.approx(var)
|
||||
assert mk.z == pytest.approx((mk.s - 1) / math.sqrt(var)) # 连续性校正
|
||||
assert mk.p == pytest.approx(stats.normal_two_sided(mk.z))
|
||||
tied = stats.mann_kendall([4, 4, 4, 4, 4, 4]) # 全是并列:方差为 0
|
||||
assert tied.s == 0 and tied.p == 1.0
|
||||
small_tied = stats.mann_kendall([1, 1, 2, 3]) # n ≤ 10 但有并列:也走正态近似
|
||||
assert not small_tied.exact
|
||||
|
||||
|
||||
def test_theil_sen_median_mad_and_rank():
|
||||
xs = [Fraction(i) for i in range(4)]
|
||||
assert stats.theil_sen(xs, [Fraction(1), Fraction(3), Fraction(2), Fraction(6)]) == Fraction(19, 12)
|
||||
assert stats.theil_sen([Fraction(0)], [Fraction(1)]) is None
|
||||
assert stats.median([Fraction(3), Fraction(1), Fraction(2)]) == 2
|
||||
assert stats.median([Fraction(1), Fraction(2), Fraction(3), Fraction(10)]) == Fraction(5, 2)
|
||||
assert stats.median([]) is None
|
||||
assert stats.mad([Fraction(v) for v in (1, 1, 2, 2, 4, 6, 9)]) == 1
|
||||
assert stats.rank_desc(5, [1, 7, 5, 9]) == 3 and stats.rank_desc(10, [1, 2]) == 1
|
||||
|
||||
|
||||
def test_theil_sen_on_many_points_matches_the_plain_definition():
|
||||
rng = random.Random(9)
|
||||
for n in (2, 3, 11, 60):
|
||||
xs = [Fraction(i) for i in range(n)]
|
||||
ys = [Fraction(rng.randint(-500, 500), rng.choice([1, 2, 3, 7, 100])) for _ in range(n)]
|
||||
slopes = sorted((ys[j] - ys[i]) / (xs[j] - xs[i]) for i in range(n) for j in range(i + 1, n))
|
||||
m = len(slopes)
|
||||
expected = slopes[m // 2] if m % 2 else (slopes[m // 2 - 1] + slopes[m // 2]) / 2
|
||||
assert stats.theil_sen(xs, ys) == expected
|
||||
assert stats.theil_sen([0.0, 1.0, 2.0], [1.0, 3.0, 2.0]) == 0.5 # 浮点照样算
|
||||
# 浮点分不开的斜率:精确地比
|
||||
tiny = Fraction(1, 10 ** 25)
|
||||
ys = [Fraction(0), Fraction(1) + tiny, Fraction(2) - tiny, Fraction(3)]
|
||||
xs = [Fraction(i) for i in range(4)]
|
||||
slopes = sorted((ys[j] - ys[i]) / (xs[j] - xs[i]) for i in range(4) for j in range(i + 1, 4))
|
||||
assert stats.theil_sen(xs, ys) == (slopes[2] + slopes[3]) / 2
|
||||
|
||||
|
||||
def test_the_fast_median_of_many_values_is_still_exact():
|
||||
rng = random.Random(5)
|
||||
for size in (513, 514, 2001):
|
||||
# 浮点分不开的值:1/3 加上远小于浮点精度的扰动
|
||||
values = [Fraction(1, 3) + Fraction(rng.randint(-3, 3), 10 ** 30) for _ in range(size)]
|
||||
values += [Fraction(rng.randint(-1000, 1000), rng.randint(1, 260)) for _ in range(size)]
|
||||
ordered = sorted(values)
|
||||
n = len(ordered)
|
||||
expected = ordered[n // 2] if n % 2 else (ordered[n // 2 - 1] + ordered[n // 2]) / 2
|
||||
assert stats.median(values) == expected
|
||||
Reference in New Issue
Block a user