Merge branch 'an/w1' into an1/d

This commit is contained in:
ThearchyHelios
2026-10-21 13:13:11 +08:00
26 changed files with 6096 additions and 23 deletions
+145 -6
View File
@@ -8,14 +8,24 @@
- recompute:按结论里记下的 params、settings、probes 和输入快照(按 sha256 取回)重算一遍,不现查;和封存的用
contract.same 比。
契约(名字、形状、缺省阈值)在 contract.py。
契约(名字、形状、缺省阈值)在 contract.py。各算子在 decompose.py(变化拆解)、series.py(波动基线、趋势)、differ.py
(差异检验)、comparable.py(可比性检查);统计在 stats.py,判定语和数的写法在 phrases.py,现查在 probe.py。
**一份结论只由 inputs(快照里的格)、probes、params、settings、method 决定**:run 时从 context 读到的东西(运行当天、目录
里的度量类型和叫法、现查的范围)都记进结论——运行当天、目录事实在 params(today、catalog),现查结果在 probes——复算只读
记下的。同样的输入两次 run 逐字节相同(现查结果相同时);recompute 和 run 逐字段相同。
**参数**(AnalysisRequest.params;列名不分大小写,按快照认)见各算子文件头;params 里的 catalog、today、depth 由系统填,
调用方写了也不收。
"""
from __future__ import annotations
from collections.abc import Callable, Mapping
from datetime import date, datetime
from typing import Any
from app.analysis.contract import AnalysisRequest, AnalysisResult, Context, InputSnapshot
from app.analysis.contract import (MODES, OPS, AnalysisRequest, AnalysisResult, CallError, Context, InputSnapshot,
effective_settings)
#: 按调用里写的名字取回输入快照;没有这个名字返回 None
Resolver = Callable[[str], InputSnapshot | None]
@@ -23,16 +33,145 @@ Resolver = Callable[[str], InputSnapshot | None]
ParentResolver = Callable[[str], AnalysisResult | None]
def _op_mode(request: Mapping[str, Any]) -> tuple[str, str | None]:
from app.analysis.phrases import OP_NAMES
op = request.get("op")
if op not in OPS:
raise CallError("unknown_op", f"不认识的分析:{op}。能做的有{'、'.join(OP_NAMES[o] for o in OPS)}")
modes = MODES.get(op)
mode = request.get("mode")
if modes is None:
return op, None
if mode not in modes:
raise CallError("unknown_mode", f"{OP_NAMES[op]}要写模式:{' / '.join(modes)}")
return op, mode
def _compute(op: str, mode: str | None, raw: Mapping[str, Any], env: Any) -> AnalysisResult:
from app.analysis import comparable, decompose, differ, series
if op == "decompose":
return {"additive": decompose.additive, "ratio": decompose.ratio, "factor": decompose.factor}[mode](raw, env) # type: ignore[index]
if op == "baseline":
return series.baseline(raw, env)
if op == "trend":
return series.trend(raw, env)
if op == "differ":
return differ.differ(raw, env)
return comparable.comparable(raw, env)
def _env(op: str, mode: str | None, settings: Mapping[str, Any], inputs: Any, facts: Any, book: Any,
today: date | None, parents: ParentResolver | None, hypothesis: Any) -> Any:
from app.analysis.common import Env
return Env(op=op, mode=mode, settings=settings, inputs=inputs, facts=facts, book=book, today=today,
parents=parents, hypothesis=hypothesis if isinstance(hypothesis, str) else None)
async def run(request: AnalysisRequest, resolve: Resolver, *, settings: Mapping[str, Any], context: Context,
parents: ParentResolver | None = None) -> AnalysisResult:
"""算一份分析结论。settings 是 contract.effective_settings 算好的阈值。"""
raise NotImplementedError("analysis.run:第 1 波 W1-a 实现")
from app.analysis.common import DERIVED_PARAMS
from app.analysis.facts import Facts
from app.analysis.inputs import Inputs
from app.analysis.probe import MAX_ROUNDS, Book, fetch
op, mode = _op_mode(request)
raw = {k: v for k, v in (request.get("params") or {}).items() if k not in DERIVED_PARAMS}
full = effective_settings(settings)
prober = context.get("probe")
today = context.get("today") or date.today()
if isinstance(today, datetime):
today = today.date()
def dialect(name: str, snap: Mapping[str, Any]) -> str | None:
given = snap.get("dialect")
if isinstance(given, str) and given:
return given
lookup = getattr(prober, "dialect", None)
return lookup(str(snap.get("source") or "")) if lookup is not None else None
known: dict[tuple[str, ...], Any] = {}
for rounds in range(MAX_ROUNDS + 1):
book = Book(known)
env = _env(op, mode, full, Inputs(resolve, dialect), Facts(catalogs=context.get("catalogs")), book, today,
parents, request.get("hypothesis"))
result = _compute(op, mode, raw, env)
missing = book.missing
if not missing or rounds == MAX_ROUNDS:
return result
known.update(await fetch(book, context))
return result # pragma: no cover - 循环里一定返回
def recompute(result: AnalysisResult, load: Callable[[str], InputSnapshot | None], *,
load_result: Callable[[str], AnalysisResult | None] | None = None) -> AnalysisResult:
"""复算:load 按 sha256 取回输入快照,load_result 按 sha256 取回 parent 那份结论。"""
raise NotImplementedError("analysis.recompute:第 1 波 W1-a 实现")
"""复算:load 按 sha256 取回输入快照,load_result 按 sha256 取回 parent 那份结论。用结论里记下的 params(含运行当天、
目录事实)、settings、probes,不现查、不读目录。取不回输入时抛 CallError(code input_missing);方法升版后调用不再成立
时同样抛 CallError(照 run 的说法)。和封存的那份用 contract.same 比。"""
from app.analysis.common import DERIVED_PARAMS
from app.analysis.facts import Facts
from app.analysis.inputs import _CELL, Inputs
from app.analysis.probe import Book
op, mode = _op_mode(result)
params = dict(result.get("params") or {})
settings = effective_settings(result.get("settings"))
names: dict[str, tuple[str | None, str | None, list[str]]] = {}
parents_at: dict[str, str] = {}
for item in result.get("inputs") or []:
ref = str(item.get("ref") or "")
if item.get("role") == "parent":
parents_at[ref] = str(item.get("artifact") or "")
continue
cell = _CELL.match(ref) if "row" in item else None
name = cell.group("name") if cell else ref
names.setdefault(name, (item.get("artifact"), item.get("dialect"), list(item.get("masked") or [])))
def resolve(name: str) -> InputSnapshot | None:
artifact, _, masked = names.get(name, (None, None, []))
found = load(artifact) if artifact else None
if not isinstance(found, Mapping):
raise CallError("input_missing", f"复算取不回 {name} 的查询快照")
return {**found, "artifact": artifact, "masked": masked} # type: ignore[typeddict-item]
def dialect(name: str, snap: Mapping[str, Any]) -> str | None:
return names.get(name, (None, None, []))[1]
def parents(name: str) -> AnalysisResult | None:
artifact = parents_at.get(name)
found = load_result(artifact) if artifact and load_result is not None else None
if not isinstance(found, Mapping):
raise CallError("input_missing", f"复算取不回上一层的分析结论 {name}")
return found # type: ignore[return-value]
today = date.fromisoformat(params["today"]) if isinstance(params.get("today"), str) else None
env = _env(op, mode, settings, Inputs(resolve, dialect), Facts(recorded=params.get("catalog") or {}),
Book(result.get("probes") or []), today, parents, result.get("hypothesis"))
raw = {k: v for k, v in params.items() if k not in DERIVED_PARAMS}
return _compute(op, mode, raw, env)
__all__ = ["ParentResolver", "Resolver", "recompute", "run"]
def exhausted(settings: Mapping[str, Any]) -> CallError:
"""本运行分析次数用完(max_calls)的调用错误:入口数着次数,到了上限交回它,不再算。"""
limit = int(effective_settings(settings)["max_calls"])
return CallError("max_calls", f"这次运行的分析次数已经用完(最多 {limit} 次):用已有的分析结论写,不再新做分析")
def make_prober(*args: Any, **kwargs: Any) -> Any:
"""见 app/analysis/probe.make_prober。"""
from app.analysis.probe import make_prober as make
return make(*args, **kwargs)
def comparable_reasons(result: Mapping[str, Any]) -> list[str]:
"""可比性结论里 fail、warn 的原因,一句一个(口径卡指标的 comparability.reasons)。"""
from app.analysis.comparable import reasons
return reasons(result)
__all__ = ["ParentResolver", "Resolver", "comparable_reasons", "exhausted", "make_prober", "recompute", "run"]
+136
View File
@@ -0,0 +1,136 @@
"""算子共用的:一次计算的环境(输入、目录事实、现查簿、阈值)、结论的组装、几样参数的规范化。
每个算子是 (raw params, Env) → AnalysisResult 的纯函数:只经 Env 取输入快照(Inputs)、目录事实(Facts)、现查结果
(Book),阈值只来自 Env.settings。run 和 recompute 只是装 Env 的方式不同(见 app/analysis/__init__.py)。
"""
from __future__ import annotations
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from datetime import date
from fractions import Fraction
from typing import Any
from app.analysis import phrases as ph
from app.analysis.contract import METHODS, AnalysisResult, CallError
from app.analysis.facts import Facts
from app.analysis.inputs import Inputs
from app.analysis.numbers import to_fraction
from app.analysis.probe import Book
#: 结论的 params 里由系统填、不收调用方的几项:目录事实、运行当天、下钻的层数
DERIVED_PARAMS = frozenset({"catalog", "today", "depth"})
@dataclass
class Env:
op: str
mode: str | None
settings: Mapping[str, Any]
inputs: Inputs
facts: Facts
book: Book
today: date | None
parents: Callable[[str], Mapping[str, Any] | None] | None
hypothesis: str | None
def threshold(self, key: str) -> Fraction:
value = to_fraction(self.settings[key])
assert value is not None, key
return value
def label_for(env: Env, raw: Any, snap: Any, column: str | None, *, default: str | None = None) -> str:
"""判定语里的度量叫法:agent 给的 label → 目录里的中文名 → default → 列名,依次取第一个消毒后不为空的(agent 给了能用的
就不去查目录)。"""
if ph.clean_label(raw):
return ph.clean_label(raw)
catalog = env.facts.label(snap, column) if column else None
return ph.pick_label(catalog, default, column)
def column_label(env: Env, snap: Any, column: str) -> str:
"""维度、因子这类列的叫法:目录里的中文名,没有就是列名(都消毒)。"""
return ph.pick_label(env.facts.label(snap, column), column)
def today_param(env: Env) -> str:
"""要判本期完不完整的算子把运行当天记进 params(复算用记下的那一天)。"""
day = env.today or date.today()
env.today = day
return day.isoformat()
def build(env: Env, params: dict[str, Any], used: Sequence[str], *, status: str = "ok",
reason_code: str | None = None, reason: str | None = None, table: Mapping[str, Any] | None = None,
values: Mapping[str, Any] | None = None, verdict: Mapping[str, Any] | None = None, criterion: str,
phrase: str, caveats: Sequence[Mapping[str, str]] = ()) -> AnalysisResult:
"""组装一份结论:形状严格按 contract.AnalysisResult,params 补上查过的目录事实。"""
record = env.facts.record()
out_params = {k: v for k, v in params.items() if v is not None}
if record:
out_params["catalog"] = record
return {
"op": env.op,
"mode": env.mode,
"method": METHODS[(env.op, env.mode)],
"params": out_params,
"settings": {k: env.settings[k] for k in used},
"inputs": list(env.inputs.records),
"probes": env.book.used,
"status": status, # type: ignore[typeddict-item]
"reason_code": reason_code,
"reason": reason,
"table": dict(table) if table is not None else None, # type: ignore[typeddict-item]
"values": dict(values or {}),
"verdict": dict(verdict) if verdict is not None else None, # type: ignore[typeddict-item]
"criterion": criterion,
"phrase": phrase,
"caveats": [dict(c) for c in caveats], # type: ignore[misc]
"hypothesis": env.hypothesis,
}
def refuse(env: Env, params: dict[str, Any], used: Sequence[str], code: str, reason: str, *,
criterion: str, caveats: Sequence[Mapping[str, str]] = ()) -> AnalysisResult:
"""算不了(status refused):判定语是「{算子的叫法}没有做成:{原因}」。"""
return build(env, params, used, status="refused", reason_code=code, reason=reason, criterion=criterion,
phrase=ph.refused(env.op, reason), caveats=caveats)
def need(raw: Mapping[str, Any], key: str, what: str) -> Any:
"""必填的参数;没写是调用错误。"""
value = raw.get(key)
if value is None or (isinstance(value, str) and not value.strip()) or value == []:
raise CallError("missing_param", f"没写{what}")
return value
def written_places(raw: Any) -> int:
"""快照里一格写到第几位小数(headline 是取整过的,按它的书写精度容差):数字文本按字面,浮点按最短写法,整数为 0。"""
if isinstance(raw, bool) or raw is None:
return 0
text = repr(raw) if isinstance(raw, float) else str(raw).strip()
if "e" in text.lower():
return 0
return len(text.split(".", 1)[1]) if "." in text else 0
def closes(total: Fraction, cell: Fraction, raw: Any) -> bool:
"""各段之和(或合起来的比率)和总数那一格对得上:十进制逐位相等;总数取整过的,差在它书写精度的半个末位以内也算;浮点
算出来的差在最后几位的也算(相对误差 1e-9 以内,同 engine/compare._equal)。"""
if total == cell:
return True
gap = abs(total - cell)
if gap <= Fraction(1, 2) / Fraction(10) ** written_places(raw):
return True
scale = max(abs(total), abs(cell))
return scale != 0 and gap <= scale * Fraction(1, 10 ** 9)
def sign_word(value: Fraction | float | int) -> str | None:
return "up" if value > 0 else "down" if value < 0 else None
__all__ = ["DERIVED_PARAMS", "Env", "build", "closes", "column_label", "label_for", "need", "refuse", "sign_word",
"today_param", "written_places"]
+540
View File
@@ -0,0 +1,540 @@
"""可比性检查(细案 01 §6):比两期之前先确认能比。三处用到:agent 单独调用(analysis_comparable);变化拆解给了两份快照时
先做;口径卡跨两份查询算环比、同比时自动做(入口经 run 调)。波动基线、趋势判本期完整用这里的 span_coverage。
五项检查,每项 pass / warn / fail / unknown(unknown 只写「核对不了」,不判不可比):
- 同口径:两份 SQL 一字不差 → pass;规范化(data/sql_semantics.normalize_sql)后只差字面量:把时间条件的界(data/coverage
认出的时间谓词的界,整个表达式)和像日期的字面量换成同一个占位再比,一样就是只差日期 → pass,不一样就是比的两段不是
两期 → fail「两边除了日期,筛选的还不一样」;规范形式不同交给口径账本的 explain_sql_diff,等价 → pass,只差时间范围、排序
→ pass,否则 fail,写出差在哪些方面(只写方面,不写 SQL 原文里的值)。两份来自不同的数据源 fail。同一份快照里按期次分组
的天然同口径。
- 窗口长度:用 data/coverage 的 filters 取业务日期列(目录的 business_date;没写就取唯一那一列时间谓词)上的区间,界是字面
量的直接解析,按运行当天现算的在库里算(Book 的 bound,记进 probes);算两边天数。整月对整月、整季对整季算一样长。
不相等:度量是流量 fail,比率、存量、没标的 warn。按期次分组的:两期粒度相同,一样长。
- 整期:窗口(或这一期)的终点和表里这一列的实际最大值(Book 的 range:现查,查不成退回剖析事实)、运行当天比;数据从
窗口起点之后才开始的也算不完整。不完整:流量 fail(「本期数据只到 2026-10-15,还差 3 天」),其余 warn;表的范围查不到时
只能按运行当天判,判不出的 unknown。
- 星期构成:按天的窗口、天数不是 7 的倍数时数两边的周六、周日天数,不同 warn(「两段的周末天数不同:4 天对 6 天」)。整月、
整季、按周的不看。
- 数据问题:两边查询结果上「查询区间里这张表没有数据」(快照的 no_data)fail,SQL 检查的错误级结果(快照的 checks,level
error)warn。
判定:有 fail → incomparable;只有 warn → conditional;其余 comparable(有核对不了的项照实写在判定语后面)。
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from datetime import date, timedelta
from typing import Any
from app.analysis import periods as pd
from app.analysis import phrases as ph
from app.analysis.facts import Facts, date_source
from app.analysis.inputs import Snap
from app.analysis.probe import Book
from app.data.names import name_key
CHECKS = ("same_query", "window", "complete", "weekdays", "data")
CHECK_NAMES = {"same_query": "同口径", "window": "窗口长度", "complete": "整期", "weekdays": "星期构成",
"data": "数据问题"}
STATUS_NAMES = {"pass": "通过", "warn": "提醒", "fail": "不通过", "unknown": "核对不了"}
#: explain_sql_diff 的方面 → 叫法
_ASPECTS = {"time": "时间列", "time_range": "时间范围", "filter": "筛选条件", "aggregate": "聚合方式",
"expression": "选出来的列", "join": "关联", "tables": "查的表", "set_op": "合并方式", "group_by": "分组",
"having": "分组后的筛选", "limit": "取哪几行", "order": "排序", "parse": "写法", "other": "写法"}
#: 只差这几个方面的两份 SQL 仍是同一个量的两期
_SAME_ASPECTS = frozenset({"time_range", "order"})
#: 一项 SQL 检查的说明最多引多少字
_CHECK_TEXT_MAX = 80
@dataclass
class Check:
id: str
status: str = "unknown"
reasons: list[str] = field(default_factory=list)
base: str | None = None
target: str | None = None
@dataclass
class Span:
"""一边的时间范围:start / end 是窗口(或这一期)的头尾(含);table / column 是这段时间按的那一列(探查表的范围用);
sql_window 是 SQL 时间条件在这一列上的区间(按期次分组时看它有没有截掉这一期)。"""
label: str
snap: Snap
start: date | None = None
end: date | None = None
table: str | None = None
column: str | None = None
sql_window: Any = None
unit: str | None = None # 这一段是不是整月、整季、整年(calendar_unit),按期次时是粒度
@dataclass
class Outcome:
checks: list[Check]
verdict: str
reasons: list[str]
unknown: list[str]
phrase: str
structure: str | None # 同口径不通过的原因(变化拆解据此不拆)
base: Span | None = None
target: Span | None = None
def check(self, cid: str) -> Check:
return next(c for c in self.checks if c.id == cid)
# --------------------------------------------------------------------------
# 同口径
# --------------------------------------------------------------------------
def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None:
"""时间条件的界、像日期的字面量换成同一个占位之后的规范形式(文本 + 剩下的字面量)。解析不了为 None。"""
from sqlglot import exp
from app.data import coverage
from app.data.sql_semantics import normalize_sql
tree = coverage.parse_sql(sql, read)
if tree is None:
return None
for flt in coverage.filters(tree):
for _, node, _ in flt.conds:
if node.parent is not None:
node.replace(exp.Literal.string("__day__"))
for lit in list(tree.find_all(exp.Literal)):
if coverage.literal_bound(lit)[1] is not None:
lit.replace(exp.Literal.string("__day__"))
normalized = normalize_sql(tree.sql(dialect=read), read)
if normalized.parse_error:
return None
return normalized.text, normalized.literals
def same_query(base: Snap, target: Snap) -> Check:
from app.data.sql_semantics import explain_sql_diff, normalize_sql
check = Check("same_query")
if base.source != target.source:
check.status, check.reasons = "fail", ["两份查询来自不同的数据源"]
return check
a, b = base.sql.strip(), target.sql.strip()
if a == b:
check.status = "pass"
return check
read = base.dialect or target.dialect
na, nb = normalize_sql(a, read), normalize_sql(b, read)
if na.parse_error or nb.parse_error:
check.reasons = ["SQL 解析不了"]
return check
if na.text == nb.text:
ua, ub = _undated(a, read), _undated(b, read)
if ua is None or ub is None:
return check
if ua == ub:
check.status = "pass"
else:
check.status, check.reasons = "fail", ["两边除了日期,筛选的还不一样"]
return check
try:
diff = explain_sql_diff(a, b, read)
except Exception: # noqa: BLE001 - 比不了就当核对不了
return check
aspects = [d.aspect for d in diff.items]
if diff.equivalent or (aspects and set(aspects) <= _SAME_ASPECTS):
check.status = "pass"
return check
names = list(dict.fromkeys(_ASPECTS.get(x, "写法") for x in aspects if x not in _SAME_ASPECTS)) or ["写法"]
check.status, check.reasons = "fail", [f"两份查询的口径结构不同(差在{'、'.join(names)})"]
return check
# --------------------------------------------------------------------------
# 窗口
# --------------------------------------------------------------------------
def _bound_days(snap: Snap, conds: Sequence[tuple[str, Any, bool]], book: Book) -> dict[int, Any]:
from app.data import coverage
days: dict[int, Any] = {}
for _, node, _ in conds:
literal, day = coverage.literal_bound(node)
if literal:
days[id(node)] = day
continue
try:
text = node.sql(dialect=snap.dialect)
except Exception: # noqa: BLE001
days[id(node)] = None
continue
probe = book.bound(snap.source, text)
days[id(node)] = coverage.as_day(probe.get("value")) if probe else None
return days
def _filters_on(snap: Snap, measure: str | None, facts: Facts,
column: tuple[str, str] | None = None) -> tuple[Any, str | None]:
"""SQL(收窄到度量那一项)里按业务日期列(或唯一那一列时间谓词,或给定的那一列)的时间条件:(那段条件, 认不出的原因)。"""
from app.data import coverage
tree = coverage.parse_sql(snap.sql, snap.dialect)
if tree is None:
return None, "SQL 解析不了"
narrowed = coverage.narrowed(tree, measure, snap.dialect) if measure else tree
flts = coverage.filters(narrowed)
groups: dict[tuple[str, str], list[Any]] = {}
for flt in flts:
groups.setdefault((name_key(flt.table_name), name_key(flt.column_name)), []).append(flt)
if column is not None:
wanted = (name_key(column[0]), name_key(column[1]))
wanted_short = (name_key(column[0].rsplit(".", 1)[-1]), wanted[1])
chosen = groups.get(wanted) or next((v for k, v in groups.items()
if (k[0].rsplit(".", 1)[-1], k[1]) == wanted_short), None)
if chosen is None:
return None, None
else:
if not groups:
return None, "SQL 里没有时间条件"
dated = [v for v in groups.values()
if (bd := facts.business_date(snap.source, v[0].table_name)) and name_key(bd) == name_key(v[0].column_name)]
if len(dated) == 1:
chosen = dated[0]
elif len(groups) == 1:
chosen = next(iter(groups.values()))
else:
return None, "时间条件不止一列,认不出按哪一列算"
if len({f.key for f in chosen}) > 1:
return None, "这一列上有好几段时间条件"
return chosen[0], None
def sql_span(label: str, snap: Snap, measure: str | None, facts: Facts, book: Book) -> tuple[Span, str | None]:
"""一份查询的时间窗口(两份快照比两期时)。"""
from app.data import coverage
span = Span(label, snap)
flt, why = _filters_on(snap, measure, facts)
if flt is None:
return span, why
if flt.column_name.lower() in snap.masked:
return span, "时间列是遮罩列" # 遮罩列不探表的范围
span.table, span.column = flt.table_name, flt.column_name
window = coverage.window_of(flt.conds, _bound_days(snap, flt.conds, book))
if window is None or window.start is None or window.end is None:
return span, "时间窗口没有两头"
span.start, span.end = window.start, window.end
span.unit = pd.calendar_unit(window.start, window.end)
return span, None
def period_span(label: str, snap: Snap, period: Any, item: pd.Period, grain: str, measure: str | None,
facts: Facts, book: Book) -> Span:
"""按期次分组的一份快照里的一期:这一期覆盖的日子,加上 SQL 在期次那一列上的时间条件。"""
from app.data import coverage
span = Span(label, snap, item.start, item.end, unit=grain)
found = date_source(snap.sql, period)
if found is None:
return span
span.table, span.column = found
if span.column.lower() in snap.masked:
span.table = span.column = None
return span
flt, _ = _filters_on(snap, measure, facts, column=found)
if flt is not None:
span.sql_window = coverage.window_of(flt.conds, _bound_days(snap, flt.conds, book))
return span
def _range(span: Span, book: Book) -> tuple[bool, date | None, date | None, bool]:
"""(查得到, 最早, 最晚, 一个值都没有):表里这一列的实际范围。"""
from app.data import coverage
if not (span.table and span.column):
return False, None, None, False
probe = book.range(span.snap.source, span.table, span.column)
if probe is None:
return False, None, None, False
lo = coverage.as_day(probe.get("min")) if probe.get("min") else None
hi = coverage.as_day(probe.get("max")) if probe.get("max") else None
empty = probe.get("basis") == "live" and probe.get("min") is None and probe.get("max") is None
return True, lo.day if lo else None, hi.day if hi else None, empty
def span_coverage(span: Span, today: date, book: Book) -> tuple[str, str | None]:
"""这一段(窗口或一期)的数据齐不齐:(pass / short / unknown, 不齐的那一句)。没过完(终点不早于运行当天:当天的数据还没
到齐)、SQL 的时间条件截掉了一部分、表里的数据没到(最晚早于终点)或来得晚(最早晚于起点),都算不齐。"""
if span.start is None or span.end is None:
return "unknown", None
lo, hi = span.start, span.end
window = span.sql_window
if window is not None:
if window.start is not None:
lo = max(lo, window.start)
if window.end is not None:
hi = min(hi, window.end)
if span.end >= today:
hi = min(hi, today - timedelta(days=1))
known, dmin, dmax, empty = _range(span, book)
if empty:
return "short", f"{span.label}在 {span.start.isoformat()} 到 {span.end.isoformat()} 没有数据"
if dmin is not None:
lo = max(lo, dmin)
if dmax is not None:
hi = min(hi, dmax)
if hi < lo:
return "short", f"{span.label}在 {span.start.isoformat()} 到 {span.end.isoformat()} 没有数据"
parts = []
if hi < span.end:
parts.append(f"{span.label}数据只到 {hi.isoformat()},还差 {(span.end - hi).days} 天")
if lo > span.start:
parts.append(f"{span.label}数据从 {lo.isoformat()} 才开始,缺前 {(lo - span.start).days} 天")
if parts:
return "short", ",".join(parts)
return ("pass", None) if known and dmax is not None else ("unknown", None)
def _days(span: Span) -> int | None:
return (span.end - span.start).days + 1 if span.start and span.end else None
def _shown(span: Span) -> str | None:
if span.start is None or span.end is None:
return None
if span.start == span.end:
return f"{span.start.isoformat()}(1 天)"
return f"{span.start.isoformat()} 到 {span.end.isoformat()}({_days(span)} 天)"
# --------------------------------------------------------------------------
# 五项检查和判定
# --------------------------------------------------------------------------
def _window_check(base: Span, target: Span, flow: bool, *, grouped: bool) -> Check:
check = Check("window", base=_shown(base), target=_shown(target))
a, b = _days(base), _days(target)
if a is None or b is None:
return check
if a == b or grouped or (base.unit and base.unit == target.unit):
check.status = "pass"
return check
check.status = "fail" if flow else "warn"
check.reasons = [f"两段的天数不同:{a} 天对 {b} 天"]
return check
def _complete_check(sides: Sequence[Span], today: date, book: Book, flow: bool) -> Check:
check = Check("complete")
shorts, unknown = [], False
for span in sides:
status, reason = span_coverage(span, today, book)
if status == "short":
shorts.append(reason)
elif status == "unknown":
unknown = True
if span.label == "基期":
check.base = reason or STATUS_NAMES[status if status != "short" else "fail"]
else:
check.target = reason or STATUS_NAMES[status if status != "short" else "fail"]
if shorts:
check.status, check.reasons = ("fail" if flow else "warn"), [r for r in shorts if r]
elif not unknown:
check.status = "pass"
return check
def _weekday_check(base: Span, target: Span, *, grain: str | None) -> Check:
check = Check("weekdays")
a, b = _days(base), _days(target)
if a is None or b is None:
return check
if grain in ("week", "month", "quarter", "seq") or (a % 7 == 0 and b % 7 == 0) or \
(base.unit and base.unit == target.unit and base.unit in ("month", "quarter", "year")):
check.status = "pass"
return check
wa, wb = pd.weekend_days(base.start, base.end), pd.weekend_days(target.start, target.end) # type: ignore[arg-type]
check.base, check.target = f"周末 {wa} 天", f"周末 {wb} 天"
if wa == wb:
check.status = "pass"
else:
check.status, check.reasons = "warn", [f"两段的周末天数不同:{wa} 天对 {wb} 天"]
return check
def _data_check(sides: Sequence[tuple[str, Snap]], measure: str | None) -> Check:
from app.data import coverage
check = Check("data", status="pass")
fails, warns = [], []
for label, snap in sides:
found = coverage.no_data_of(snap.data, measure) if measure else \
next(iter(snap.data.get("no_data") or []), None)
if found:
fails.append(f"{label}查询的区间里这张表没有数据")
errors = [c for c in snap.data.get("checks") or [] if isinstance(c, Mapping) and c.get("level") == "error"]
if errors:
text = str(errors[0].get("message") or "").strip().replace(";", ",")
if len(text) > _CHECK_TEXT_MAX:
text = text[:_CHECK_TEXT_MAX - 1] + "…"
warns.append(f"{label}查询的 SQL 检查有错误:{text}" if text else f"{label}查询的 SQL 检查有错误")
if fails:
check.status = "fail"
elif warns:
check.status = "warn"
check.reasons = fails + warns
return check
def _decide(checks: list[Check], base: Span | None, target: Span | None) -> Outcome:
fails = [r for c in checks if c.status == "fail" for r in c.reasons]
warns = [r for c in checks if c.status == "warn" for r in c.reasons]
unknown = [CHECK_NAMES[c.id] for c in checks if c.status == "unknown"]
verdict = "incomparable" if fails else "conditional" if warns else "comparable"
reasons = fails + warns
structure = next((c.reasons[0] for c in checks if c.id == "same_query" and c.status == "fail" and c.reasons), None)
return Outcome(checks, verdict, reasons, unknown, ph.comparable_phrase(verdict, reasons, unknown), structure,
base, target)
def compare_queries(base: Snap, target: Snap, measure: str | None, facts: Facts, book: Book, today: date) -> Outcome:
"""两份快照比两期(本周一份、上周一份)。measure 是两边的度量列(收窄 SQL 找时间条件、认度量类型);没有为 None。"""
same = same_query(base, target)
b_span, _ = sql_span("基期", base, measure, facts, book)
t_span, _ = sql_span("本期", target, measure, facts, book)
flow = bool(measure) and facts.measure(target, measure)[0] == "flow" # type: ignore[arg-type]
checks = [same, _window_check(b_span, t_span, flow, grouped=False),
_complete_check([b_span, t_span], today, book, flow), _weekday_check(b_span, t_span, grain=None),
_data_check([("基期", base), ("本期", target)], measure)]
return _decide(checks, b_span, t_span)
def compare_periods(snap: Snap, period: str, base: pd.Period, target: pd.Period, grain: str, measure: str | None,
facts: Facts, book: Book, today: date) -> Outcome:
"""同一份快照里按期次分组的两期:天然同口径;窗口按期次的粒度;整期看这两期各自齐不齐。"""
same = Check("same_query", status="pass")
b_span = period_span("基期", snap, period, base, grain, measure, facts, book)
t_span = period_span("本期", snap, period, target, grain, measure, facts, book)
flow = bool(measure) and facts.measure(snap, measure)[0] == "flow" # type: ignore[arg-type]
if grain == "seq":
checks = [same, Check("window"), Check("complete"), Check("weekdays"), _data_check([("", snap)], measure)]
else:
checks = [same, _window_check(b_span, t_span, flow, grouped=True),
_complete_check([b_span, t_span], today, book, flow),
_weekday_check(b_span, t_span, grain=grain), _data_check([("", snap)], measure)]
return _decide(checks, b_span, t_span)
def table_of(outcome: Outcome) -> dict[str, Any]:
"""检查表(结论的 table):检查项、结果、基期、本期、说明。"""
rows = [[CHECK_NAMES[c.id], STATUS_NAMES[c.status], c.base, c.target, ";".join(c.reasons) or None]
for c in outcome.checks]
return {"columns": ["check", "status", "base", "target", "note"],
"labels": {"check": "检查项", "status": "结果", "base": "基期", "target": "本期", "note": "说明"},
"rows": rows}
def values_of(outcome: Outcome) -> dict[str, Any]:
out: dict[str, Any] = {c.id: c.status for c in outcome.checks}
for side, span in (("base", outcome.base), ("target", outcome.target)):
if span is not None and span.start and span.end:
out[f"{side}_start"] = span.start.isoformat()
out[f"{side}_end"] = span.end.isoformat()
out[f"{side}_days"] = str(_days(span))
out[f"{side}_weekend"] = str(pd.weekend_days(span.start, span.end))
return out
def reasons(result: Mapping[str, Any]) -> list[str]:
"""一份可比性结论里 fail、warn 的原因,一句一个(口径卡指标的 comparability.reasons 用)。按检查表的说明拆开。"""
out: list[str] = []
table = result.get("table") or {}
for row in table.get("rows") or []:
if len(row) >= 5 and row[1] in (STATUS_NAMES["fail"], STATUS_NAMES["warn"]) and row[4]:
out.extend(str(row[4]).split(";"))
return out
# --------------------------------------------------------------------------
# 算子:analysis_comparable
# --------------------------------------------------------------------------
def comparable(raw: Mapping[str, Any], env: Any) -> Any:
"""可比性检查这个算子。参数:两份快照 base_query、target_query;或一份快照 query + period(期次列)+ base、target(期次
列里的两个值)、可选 grain。都可以写 measure(度量列:收窄 SQL 找时间条件、按目录认它是不是流量;不写时只有一列数值列
就用它)。"""
from app.analysis.common import build, need, today_param
from app.analysis.contract import CallError
from app.analysis.inputs import same_value
params: dict[str, Any] = {}
two = raw.get("base_query") is not None or raw.get("target_query") is not None
if two:
base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号")
target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号")
if base is target:
from app.analysis.decompose import SAME_QUERY
raise CallError("same_query", SAME_QUERY)
params.update(base_query=base.ref, target_query=target.ref)
else:
base = target = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
params["query"] = base.ref
measure = target.column(raw["measure"], "度量列") if raw.get("measure") is not None else _only_number(target)
if measure is not None and two:
base.column(measure, "度量列")
params["measure"] = measure
params["today"] = today_param(env)
if two:
env.inputs.use(base, "base", [measure] if measure else [])
env.inputs.use(target, "target", [measure] if measure else [])
outcome = compare_queries(base, target, measure, env.facts, env.book, env.today)
else:
period = target.column(need(raw, "period", "期次列"), "期次列")
grain = raw.get("grain")
if grain is not None and grain not in (*pd.GRAINS, "seq"):
raise CallError("bad_grain", "粒度只能写 day、week、month、quarter,或者不写(按期次值推断)")
params.update(period=period, base=need(raw, "base", "基期(期次列里的值)"),
target=need(raw, "target", "本期(期次列里的值)"))
i = target.index(period)
values = [row[i] if i < len(row) else None for row in target.rows]
rows = {side: [r for r, v in enumerate(values) if same_value(params[side], v)] for side in ("base", "target")}
for side, what in (("base", "基期"), ("target", "本期")):
if not rows[side]:
raise CallError("period_not_found", f"期次列「{period}」里没有「{params[side]}」:{what}要写 {target.ref} "
"里有的期次值")
env.inputs.use(target, "data", [period, *([measure] if measure else [])], rows["base"] + rows["target"])
parsed = pd.parse(values, grain)
if parsed is None:
item_b = item_t = pd.Period(None, 0, None, None)
grain = "seq"
else:
items, grain = parsed
item_b, item_t = items[rows["base"][0]], items[rows["target"][0]]
params["grain"] = grain
outcome = compare_periods(target, period, item_b, item_t, grain, measure, env.facts, env.book, env.today)
caveats = []
window = outcome.check("window")
if measure and window.status == "warn" and env.facts.measure(target, measure)[0] is None:
caveats.append(ph.caveat("measure_unlabeled", ph.unlabeled_text(measure, "窗口不等只作提醒")))
verdict = {"code": outcome.verdict, "subjects": [], "direction": None}
return build(env, params, [], table=table_of(outcome), values=values_of(outcome), verdict=verdict,
criterion=ph.CRITERION_COMPARABLE, phrase=outcome.phrase, caveats=caveats)
def _only_number(snap: Snap) -> str | None:
"""没写度量列时:快照里唯一一列(没遮罩的)数值列;没有或不止一列为 None。"""
found = [c for c in snap.columns if c.lower() not in snap.masked and snap.kind(c) == "number"]
return found[0] if len(found) == 1 else None
__all__ = ["CHECKS", "CHECK_NAMES", "Check", "Outcome", "STATUS_NAMES", "Span", "comparable", "compare_periods",
"compare_queries", "period_span", "reasons", "same_query", "span_coverage", "sql_span", "table_of",
"values_of"]
+13 -1
View File
@@ -188,6 +188,12 @@ class InputSnapshot(TypedDict, total=False):
rows: list[list[Any]]
truncated: bool
masked: list[str] # 不能当维度、度量的列(遮罩列、按 SQL 认出改了名字的遮罩列),比较时不分大小写
# 以下几项照查询快照里的原样带上(第 1 波 W1-a 加,都可选):
dialect: str # SQL 的 sqlglot 方言(sqlite / mysql / postgres / oracle);不给时按探查器的 dialect(数据源名)
checks: list[dict[str, Any]] # 快照里的 SQL 检查结果(level 为 error 的算可比性的数据问题)
no_data: list[dict[str, Any]] # 快照里的「查询区间里这张表没有数据」
mask_columns: list[str] # 没给 masked 时(复算取回的是快照工件本身)按这两项认遮罩列
masked_at_query: list[str]
class AnalysisInput(TypedDict, total=False):
@@ -201,6 +207,8 @@ class AnalysisInput(TypedDict, total=False):
row: int # 一格时:第几行、哪一列、值(十进制字符串)
column: str
value: str | None
dialect: str # 读这份快照的 SQL 用的方言(第 1 波 W1-a 加,可选):复算按它读,界的写法才一样
masked: list[str] # 这份快照当时不能当维度、度量的列(小写,只记列名;第 1 波 W1-a 加,可选):复算按它认
class Probe(TypedDict, total=False):
@@ -213,6 +221,10 @@ class Probe(TypedDict, total=False):
max: str | None
basis: Literal["live", "profile", "observed"]
at: str # 现查的时刻(ISO,服务端时区)
# 按运行当天现算的界(CURRENT_DATE、date('now', …))在库里算出的值(第 1 波 W1-a 加,可选):有 expr 的这一条记的是界,
# 没有 table / column / min / max
expr: str # 界的 SQL 写法(按这份快照的方言)
value: str | None # 库里算出的日期(带时刻的写成 YYYY-MM-DD HH:MM:SS)
class Verdict(TypedDict, total=False):
@@ -284,7 +296,7 @@ class Context(TypedDict, total=False):
today: date # 运行当天(服务端时区):判本期完不完整
probe: Prober | None # None 时只用 facts
catalogs: Mapping[str, Any] # 数据源名 → 数据目录(app/data/catalog 的 notes):认度量类型、业务日期列、列的叫法
facts: Mapping[str, Any] # 数据源名 → 剖析事实(探查失败时兜底)
facts: Mapping[str, Any] # 数据源名 → 剖析事实(探查失败时兜底):{表名: last_facts 的那一项},或取它的函数
# --------------------------------------------------------------------------
+732
View File
@@ -0,0 +1,732 @@
"""变化拆解(细案 01 §2):变化从哪来。三种模式,agent 按指标的性质选,算子核对选得对不对。
**参数**(AnalysisRequest.params;列名不分大小写,按快照认):
- 一份快照:query(编号)、period(期次列)、base / target(期次列里的两个值);或两份快照:base_query、target_query(各自
只有维度列和度量列,先做可比性检查:口径结构不同的不拆,算不了 incomparable_sides;其余不可比的照样算,判定后附上
可比性结论)。
- additive:by(维度列,1–2 列)、measure(度量列);可选 headline {base, target}(基期、本期总数各一格,如 Q1.r1.gmv,用来
核对闭合)、parent {of: A3, key: [华东]}(下钻:这一层各段之和要等于 A3 里华东那一段)。
- ratio:by、numerator(分子列)、denominator(分母列);可选 headline {base, target}(整体比率的两格)、parent(上一层的比率
拆解)。
- factor:product(乘积列)、factors(因子列,2–4 个);一份快照按 period / base / target 取两行,两份快照各一行。
- 都可以写 label(度量的叫法,消毒后进判定语)。
**计算**全程 Fraction(快照里的数按字面换成精确的有理数):各段之和逐位等于总变化;比率拆解的结构效应、自身效应按中点
对称拆,Σmix + Σrate 逐位等于比率的变化;乘积拆解是 LMDI-I(对数),浮点,各因子贡献之和在浮点误差内等于总变化。判定、
阈值、边界照细案 §2.1–§2.4。
"""
from __future__ import annotations
import math
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from fractions import Fraction
from typing import Any
from app.analysis import comparable as cmp
from app.analysis import phrases as ph
from app.analysis.common import Env, build, closes, column_label, label_for, need, refuse, sign_word, today_param
from app.analysis.contract import CallError
from app.analysis.inputs import Snap, as_list, hashable, same_value
from app.analysis.numbers import canon, to_fraction
#: 辛普森:分母够大的段里,按平均权重算有这么多的段比率和整体反着变
SIMPSON_SHARE = Fraction(4, 5)
#: 下钻、总数的写法(调用错误里教 agent 怎么写)
PARENT_SHAPE = "下钻要写成 {of: A3, key: [华东]} 这样:of 是上一层变化拆解的编号,key 是那一段的维度值"
HEADLINE_SHAPE = "总数要写成 {base: Q1.r1.gmv, target: Q1.r0.gmv} 这样:base 是基期总数那一格,target 是本期总数那一格"
SAME_QUERY = "基期和本期是同一份查询:两期各给一份查询,或者给一份按期次分组的查询和期次列"
#: 乘积闭合的最小相对误差
_FACTOR_REL = Fraction(1, 10 ** 6)
_NOT_ADDITIVE = {"ratio": "比率", "identifier": "标识", "status": "状态", "attribute": "属性"}
@dataclass
class Side:
"""一期的各段:键 → 值(ratio 模式是 (分子, 分母));nulls 是这一期度量格为空的段;rows 是用到的行。"""
snap: Snap
values: dict[tuple[Any, ...], Any] = field(default_factory=dict)
nulls: list[tuple[Any, ...]] = field(default_factory=list)
rows: list[int] = field(default_factory=list)
@dataclass
class Segment:
key: tuple[Any, ...]
base: Any
target: Any
flags: list[str] = field(default_factory=list)
def _key_strings(key: Sequence[Any]) -> list[str]:
return [ph.key_text([k]) for k in key]
def _sides(raw: Mapping[str, Any], env: Env, roles: Sequence[tuple[str, str]], *, by_needed: bool = True
) -> tuple[dict[str, Any], Snap, Snap, dict[str, str], list[str]]:
"""一份或两份快照:(规范化的参数, 基期快照, 本期快照, {角色: 列名}, 维度列)。roles 是 [(参数名, 角色的叫法)]。"""
params: dict[str, Any] = {}
two = raw.get("base_query") is not None or raw.get("target_query") is not None
if two:
base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号")
target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号")
if base is target:
raise CallError("same_query", SAME_QUERY)
params.update(base_query=base.ref, target_query=target.ref)
else:
base = target = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
params["query"] = base.ref
by: list[str] = []
if by_needed:
wanted = as_list(raw.get("by"))
if not 1 <= len(wanted) <= 2:
raise CallError("bad_by", "维度列要写 1 到 2 列")
by = [target.column(c, "维度列") for c in wanted]
if two:
for c in by:
base.column(c, "维度列")
params["by"] = by
cols: dict[str, str] = {}
for key, what in roles:
if key == "factors":
continue
col = target.column(need(raw, key, what), what)
if two:
base.column(col, what)
cols[key] = col
params[key] = col
if not two:
params["period"] = target.column(need(raw, "period", "期次列"), "期次列")
params["base"] = need(raw, "base", "基期(期次列里的值)")
params["target"] = need(raw, "target", "本期(期次列里的值)")
if same_value(params["base"], params["target"]):
raise CallError("same_period", "基期和本期是同一期:写期次列里两个不同的值")
used = [*cols.values(), *([params["period"]] if "period" in params else [])]
if len(set(by)) != len(by) or set(by) & set(used) or len(set(used)) != len(used):
raise CallError("bad_by", "维度列、期次列、度量列要是不同的列")
return params, base, target, cols, by
def _period_rows(snap: Snap, period: str, wanted: Any, what: str) -> list[int]:
i = snap.index(period)
rows = [r for r, row in enumerate(snap.rows) if i < len(row) and same_value(wanted, row[i])]
if not rows:
values = list(dict.fromkeys(ph.key_text([row[i]]) for row in snap.rows if i < len(row)))
raise CallError("period_not_found", f"期次列「{period}」里没有「{wanted}」:{what}要写 {snap.ref} 里有的期次值,"
f"比如 {'、'.join(values[:6])}")
return rows
def _rows_of(raw: Mapping[str, Any], params: dict[str, Any], base: Snap, target: Snap) -> tuple[list[int], list[int]]:
if "period" in params:
return (_period_rows(base, params["period"], params["base"], "基期"),
_period_rows(target, params["period"], params["target"], "本期"))
return list(range(len(base.rows))), list(range(len(target.rows)))
def _collect(snap: Snap, rows: Sequence[int], by: Sequence[str], read: Any, labels: Sequence[str],
period_label: str | None) -> Side:
"""一期的各段:按维度键,同一键只许一行(多行是 SQL 没按这些维度汇总)。read(row) 读出这一行的值(None 是空)。"""
side = Side(snap)
seen: set[tuple[Any, ...]] = set()
idx = [snap.index(c) for c in by]
for r in rows:
row = snap.rows[r]
key = tuple(hashable(row[i] if i < len(row) else None) for i in idx)
if key in seen:
keys = "、".join([*labels, *([period_label] if period_label else [])])
raise CallError("duplicate_rows", f"{snap.ref} 里同一个{'、'.join(labels)}在同一期有不止一行:在 SQL 里按 "
f"{keys} 汇总,每段每期一行")
seen.add(key)
value = read(r)
side.rows.append(r)
if value is None:
side.nulls.append(key)
side.values[key] = value
return side
def _segments(base: Side, target: Side, zero: Any) -> list[Segment]:
"""两期的各段,按先出现的顺序:缺的一边记 0,标「新出现 / 消失」;度量格为空的记 0,标「为空」。"""
out: list[Segment] = []
for key in dict.fromkeys([*base.values, *target.values]):
flags = []
if key not in base.values:
flags.append("新出现")
if key not in target.values:
flags.append("消失")
if key in base.nulls or key in target.nulls:
flags.append("为空")
b, t = base.values.get(key), target.values.get(key)
out.append(Segment(key, zero if b is None else b, zero if t is None else t, flags))
return out
def _parent(raw: Mapping[str, Any], env: Env, mode: str, dims: int) -> tuple[dict[str, Any], list[Any], int]:
"""下钻:上一层的结论和那一段。返回 (params 里的 parent, 那一段在上一层结果表里的那一行, 这一层的层数)。"""
from app.core.artifact_store import canonical_json, content_hash
spec = raw.get("parent")
name = spec.get("of") if isinstance(spec, Mapping) else None
key = as_list(spec.get("key")) if isinstance(spec, Mapping) else []
if not isinstance(name, str) or not name.strip() or not key:
raise CallError("bad_parent", PARENT_SHAPE)
name = name.strip()
found = env.parents(name) if env.parents is not None else None
if not isinstance(found, Mapping):
raise CallError("parent_missing", f"没有叫「{name}」的分析结论:下钻写这次运行里变化拆解的编号(如 A3)")
if found.get("op") != "decompose" or found.get("mode") != mode:
raise CallError("parent_not_decompose", f"{name} 不是{'按维度' if mode == 'additive' else '比率'}的变化拆解,"
"下钻要接在同一种拆解后面")
if found.get("status") != "ok" or not isinstance(found.get("table"), Mapping):
raise CallError("parent_refused", f"{name} 没有做成,不能在它下面下钻")
depth = int((found.get("params") or {}).get("depth") or 0) + 1
limit = int(env.settings["max_drill_depth"])
if depth > limit:
raise CallError("too_deep", f"已经下钻了 {depth - 1} 层,最多 {limit} 层:换一个维度从头拆")
table = found["table"]
fixed = _FIXED_ADDITIVE if mode == "additive" else _FIXED_RATIO
columns = list(table.get("columns") or [])
dim_count = len([c for c in columns if c not in fixed])
if len(key) != dim_count:
raise CallError("bad_parent", f"{name} 按 {dim_count} 列拆,下钻的那一段要写 {dim_count} 个维度值")
hit = next((row for row in table.get("rows") or []
if all(same_value(k, v) for k, v in zip(key, row[:dim_count]))), None)
if hit is None:
raise CallError("parent_key_missing", f"{name} 里没有「{ph.key_text(key)}」这一段:写 {name} 结果表里的段")
env.inputs.parent(name, content_hash(canonical_json(found)))
return {"of": name, "key": key}, dict(zip(columns, hit)), depth
_FIXED_ADDITIVE = ("base", "target", "delta", "pct", "share", "flag")
_FIXED_RATIO = ("n_base", "n_target", "d_base", "d_target", "w_base", "w_target", "r_base", "r_target", "mix", "rate",
"total", "flag")
def _headline(raw: Mapping[str, Any], env: Env) -> tuple[dict[str, str] | None, dict[str, tuple[Fraction, Any]]]:
"""headline:基期、本期总数各一格 → ({base, target} 的写法, {base / target: (值, 原样)})。"""
spec = raw.get("headline")
if spec is None:
return None, {}
if isinstance(spec, (list, tuple)) and len(spec) == 2:
spec = {"base": spec[0], "target": spec[1]}
if not isinstance(spec, Mapping) or not spec.get("base") or not spec.get("target"):
raise CallError("bad_headline", HEADLINE_SHAPE)
out: dict[str, tuple[Fraction, Any]] = {}
refs: dict[str, str] = {}
for side, what in (("base", "基期总数"), ("target", "本期总数")):
snap, row, col, value = env.inputs.cell(spec[side], f"headline_{side}", what)
if value is None:
raise CallError("headline_empty", f"{what}那一格({snap.ref}.r{row}.{col})是空的:换一格有值的")
out[side] = (value, snap.cell(row, col))
refs[side] = f"{snap.ref}.r{row}.{col}"
return refs, out
def _measure_guard(env: Env, snap: Snap, column: str, caveats: list[dict[str, str]], tail: str) -> None:
"""additive 的度量列:目录标成(确认过的)比率、标识、状态、属性是调用错误;没标的、只是推断的照算,写附带说明。"""
kind, confirmed = env.facts.measure(snap, column)
if kind in _NOT_ADDITIVE and confirmed:
if kind == "ratio":
raise CallError("ratio_as_additive", f"「{column}」是比率(或平均值),不能按段相加:改用比率拆解,分子、分母各给一列")
raise CallError("not_a_measure", f"「{column}」在目录里是{_NOT_ADDITIVE[kind]},不是能相加的量:度量列要选金额、数量这类列")
if kind is None:
caveats.append(ph.caveat("measure_unlabeled", ph.unlabeled_text(column, tail)))
elif kind in _NOT_ADDITIVE:
caveats.append(ph.caveat("measure_unlabeled", ph.unconfirmed_text(column, kind, tail)))
def _comparability(env: Env, params: dict[str, Any], base: Snap, target: Snap, measure: str | None
) -> tuple[cmp.Outcome | None, str | None]:
"""两份快照时先做可比性:(结论, 口径结构不同的原因)。"""
if "base_query" not in params:
return None, None
params["today"] = today_param(env)
outcome = cmp.compare_queries(base, target, measure, env.facts, env.book, env.today) # type: ignore[arg-type]
return outcome, outcome.structure
def _share_rank(segments: Sequence[Any], share: Any) -> list[Any]:
"""按 share 从大到小(一样大的按先出现的)。"""
return sorted(segments, key=lambda s: -share(s))
# --------------------------------------------------------------------------
# additive
# --------------------------------------------------------------------------
def additive(raw: Mapping[str, Any], env: Env) -> Any:
used = ["main_share", "concentration", "offset_ratio", "flat_pct", "table_top_k"]
params, bs, ts, cols, by = _sides(raw, env, [("measure", "度量列")])
measure = cols["measure"]
label = label_for(env, raw.get("label"), ts, measure)
if raw.get("label") is not None:
params["label"] = ph.clean_label(raw.get("label")) or None
dims = [column_label(env, ts, c) for c in by]
caveats: list[dict[str, str]] = []
_measure_guard(env, ts, measure, caveats, "按可以按段相加处理")
refs, heads = _headline(raw, env)
if refs:
params["headline"] = refs
parent_row = None
if raw.get("parent") is not None:
params["parent"], parent_row, params["depth"] = _parent(raw, env, "additive", len(by))
used.append("max_drill_depth")
summed = not heads and parent_row is None
criterion_args = {"summed": summed}
for snap in dict.fromkeys([bs, ts]):
if snap.truncated:
env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [*by, measure])
return refuse(env, params, used, "truncated",
f"{snap.ref} 只取回了前 {len(snap.rows)} 行,各段之和不是完整的和",
criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args))
rows_b, rows_t = _rows_of(raw, params, bs, ts)
period_label = column_label(env, ts, params["period"]) if "period" in params else None
side_b = _collect(bs, rows_b, by, lambda r: bs.number(r, measure, "度量列"), dims, period_label)
side_t = _collect(ts, rows_t, by, lambda r: ts.number(r, measure, "度量列"), dims, period_label)
if bs is ts:
env.inputs.use(ts, "data", [*by, params["period"], measure], side_b.rows + side_t.rows)
else:
env.inputs.use(bs, "base", [*by, measure], side_b.rows)
env.inputs.use(ts, "target", [*by, measure], side_t.rows)
outcome, structure = _comparability(env, params, bs, ts, measure)
if structure is not None:
return refuse(env, params, used, "incomparable_sides", f"{structure},不是同一个量的两期,不拆",
criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats)
segs = _segments(side_b, side_t, Fraction(0))
if len(segs) < 2:
return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(segs)} 段,至少要 2 段才拆得开",
criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats)
return _additive_result(env, params, used, segs, by, dims, label, heads, parent_row, outcome, caveats, summed,
side_b, side_t)
def _additive_result(env: Env, params: dict[str, Any], used: list[str], segs: list[Segment], by: list[str],
dims: list[str], label: str, heads: Mapping[str, tuple[Fraction, Any]],
parent_row: Mapping[str, Any] | None, outcome: cmp.Outcome | None,
caveats: list[dict[str, str]], summed: bool, side_b: Side, side_t: Side) -> Any:
s = env.settings
main_share, concentration = env.threshold("main_share"), env.threshold("concentration")
offset_ratio, flat_pct = env.threshold("offset_ratio"), env.threshold("flat_pct")
B = sum((x.base for x in segs), Fraction(0))
T = sum((x.target for x in segs), Fraction(0))
delta = T - B
gaps: dict[str, Fraction] = {}
for side, total in (("base", B), ("target", T)):
if side in heads and not closes(total, heads[side][0], heads[side][1]):
gaps[side] = heads[side][0] - total
if parent_row is not None:
expect = to_fraction(parent_row.get(side))
if expect is None or not closes(total, expect, parent_row.get(side)):
gaps.setdefault(side, (expect or Fraction(0)) - total)
closed = not gaps
deltas = {x.key: x.target - x.base for x in segs}
shares = {k: d / delta for k, d in deltas.items()} if closed and delta != 0 else {}
gross_up = sum((d for d in deltas.values() if d > 0), Fraction(0))
gross_down = sum((d for d in deltas.values() if d < 0), Fraction(0))
opposite = abs(gross_down) if delta > 0 else gross_up if delta < 0 else gross_up
n = len(segs)
k_max = max(2, math.ceil(n / 5))
flat = delta == 0 or (B != 0 and abs(delta) / abs(B) < flat_pct)
ranked = _share_rank(segs, lambda x: shares[x.key]) if shares else []
top_k = top_k_share = None
if ranked:
running = Fraction(0)
for k, seg in enumerate(ranked, start=1):
running += shares[seg.key]
if running >= concentration:
top_k, top_k_share = k, running
break
if not closed:
code, subjects = "unclosed", []
elif flat:
code, subjects = "flat", []
elif shares[ranked[0].key] >= main_share:
code, subjects = "driver", [ranked[0]]
elif top_k is not None and 2 <= top_k <= k_max:
code, subjects = "concentrated", ranked[:top_k]
else:
code, subjects = "dispersed", []
offset = opposite > 0 and opposite >= offset_ratio * abs(delta)
if offset:
caveats.append(ph.caveat("offset", ph.offset_text(delta, gross_up, gross_down)))
nulls = [x for x in segs if "为空" in x.flags]
if nulls:
caveats.append(ph.caveat("null_values", ph.null_segments_text([ph.key_text(x.key) for x in nulls], label)))
if summed:
caveats.append(ph.caveat("headline_basis", ph.HEADLINE_BASIS))
if outcome is not None and outcome.verdict != "comparable":
caveats.append(ph.caveat("comparability", outcome.phrase))
top = ranked[0] if ranked else max(segs, key=lambda x: abs(deltas[x.key]))
total_text = ph.change(delta, B)
top_n = int(s["table_top_k"])
if code == "driver":
main = ph.decompose_driver(label, total_text, ph.key_text(top.key), ph.amount(deltas[top.key], signed=True),
shares[top.key])
elif code == "concentrated":
main = ph.decompose_concentrated(label, total_text, ph.names([ph.key_text(x.key) for x in subjects], top_n),
top_k_share)
elif code == "dispersed":
main = ph.decompose_dispersed(label, total_text, ph.dims_text(dims), main_share)
elif code == "flat":
ups = [x for x in segs if deltas[x.key] > 0]
downs = [x for x in segs if deltas[x.key] < 0]
pair = None
if offset and ups and downs:
up = max(ups, key=lambda x: deltas[x.key])
down = min(downs, key=lambda x: deltas[x.key])
pair = (ph.key_text(up.key), ph.amount(deltas[up.key], signed=True), ph.key_text(down.key),
ph.amount(deltas[down.key], signed=True))
main = ph.decompose_flat(label, delta / abs(B) if B != 0 else None, pair)
else:
gap = max(gaps.values(), key=abs)
main = ph.decompose_unclosed(ph.dims_text(dims), gap, ratio=False)
order = sorted(segs, key=lambda x: -abs(deltas[x.key]))
rows = []
for x in order:
d = deltas[x.key]
rows.append([*x.key, canon(x.base), canon(x.target), canon(d), canon(d / x.base) if x.base != 0 else None,
canon(shares[x.key]) if shares else None, "、".join(x.flags) or None])
table = {"columns": [*by, *_FIXED_ADDITIVE],
"labels": {**dict(zip(by, dims)), "base": "基期", "target": "本期", "delta": "变化", "pct": "变化率",
"share": "占变化的比重", "flag": "说明"},
"rows": rows}
values: dict[str, Any] = {
"base_total": canon(B), "target_total": canon(T), "delta_total": canon(delta),
"pct_total": canon(delta / abs(B)) if B != 0 else None, "top_key": ph.key_text(top.key),
"top_share": canon(shares[top.key]) if shares else None, "top_k": str(top_k) if top_k is not None else None,
"top_k_share": canon(top_k_share) if top_k_share is not None else None, "gross_up": canon(gross_up),
"gross_down": canon(gross_down), "segments": str(n),
}
if gaps:
values["gap"] = canon(max(gaps.values(), key=abs))
verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects],
"direction": sign_word(delta) if code in ("driver", "concentrated", "dispersed") else None}
phrase = ph.with_caveats(main, caveats, skip=("offset",) if code == "flat" else ())
return build(env, params, used, table=table, values=values, verdict=verdict,
criterion=ph.criterion_additive(s, top_k_max=k_max, summed=summed), phrase=phrase, caveats=caveats)
# --------------------------------------------------------------------------
# ratio
# --------------------------------------------------------------------------
def ratio(raw: Mapping[str, Any], env: Env) -> Any:
used = ["main_share", "flat_pct", "min_denominator", "table_top_k"]
params, bs, ts, cols, by = _sides(raw, env, [("numerator", "分子列"), ("denominator", "分母列")])
num, den = cols["numerator"], cols["denominator"]
num_label, den_label = column_label(env, ts, num), column_label(env, ts, den)
label = ph.pick_label(raw.get("label"), ph.say(num_label, "与", den_label, "之比"))
if raw.get("label") is not None:
params["label"] = ph.clean_label(raw.get("label")) or None
dims = [column_label(env, ts, c) for c in by]
refs, heads = _headline(raw, env)
if refs:
params["headline"] = refs
parent_row = None
if raw.get("parent") is not None:
params["parent"], parent_row, params["depth"] = _parent(raw, env, "ratio", len(by))
used.append("max_drill_depth")
criterion = ph.criterion_ratio(env.settings)
for snap in dict.fromkeys([bs, ts]):
if snap.truncated:
env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [*by, num, den])
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行,各段合起来不是完整的数",
criterion=criterion)
rows_b, rows_t = _rows_of(raw, params, bs, ts)
period_label = column_label(env, ts, params["period"]) if "period" in params else None
def reader(snap: Snap) -> Any:
return lambda r: (snap.number(r, num, "分子列"), snap.number(r, den, "分母列"))
side_b = _collect(bs, rows_b, by, reader(bs), dims, period_label)
side_t = _collect(ts, rows_t, by, reader(ts), dims, period_label)
for side in (side_b, side_t):
side.nulls = [k for k, v in side.values.items() if v[0] is None or v[1] is None]
side.values = {k: (v[0] or Fraction(0), v[1] or Fraction(0)) for k, v in side.values.items()}
if bs is ts:
env.inputs.use(ts, "data", [*by, params["period"], num, den], side_b.rows + side_t.rows)
else:
env.inputs.use(bs, "base", [*by, num, den], side_b.rows)
env.inputs.use(ts, "target", [*by, num, den], side_t.rows)
outcome, structure = _comparability(env, params, bs, ts, den)
if structure is not None:
return refuse(env, params, used, "incomparable_sides", f"{structure},不是同一个量的两期,不拆", criterion=criterion)
segs = _segments(side_b, side_t, (Fraction(0), Fraction(0)))
live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0]
if len(live) < 2:
return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(live)} 段,至少要 2 段才拆得开",
criterion=criterion)
for when, i in (("基期", 0), ("本期", 1)):
if sum(((x.base, x.target)[i][1] for x in live), Fraction(0)) == 0:
return refuse(env, params, used, "zero_base", ph.say(f"{when}的", den_label, "合计为 0,算不了比率"),
criterion=criterion)
for x in live:
for (n_, d_), when in ((x.base, "基期"), (x.target, "本期")):
if d_ == 0 and n_ != 0:
return refuse(env, params, used, "zero_base",
ph.say(ph.key_text(x.key), f"{when}的", den_label, "为 0 而", num_label,
"不为 0,比率拆不了"), criterion=criterion)
return _ratio_result(env, params, used, segs, by, dims, label, num_label, den_label, heads, parent_row, outcome,
criterion)
def ratio_effects(segs: Sequence[tuple[Fraction, Fraction, Fraction, Fraction]]
) -> tuple[Fraction, Fraction, list[tuple[Fraction, ...]]]:
"""比率拆解的计算(细案 §2.2,中点对称拆法)。segs 是各段 (分子_基期, 分母_基期, 分子_本期, 分母_本期),两期分母都为 0
的段先剔掉。交回 (R, R', 各段 (w, w', r, r', mix, rate)):Σmix + Σrate = R' − R 逐位成立(Σ(w' − w) = 0)。一边分母为 0
的段:那一边权重 0、比率取另一边的(自身效应为 0,变化全记在结构上)。"""
db = sum((s[1] for s in segs), Fraction(0))
dt = sum((s[3] for s in segs), Fraction(0))
R = sum((s[0] for s in segs), Fraction(0)) / db
R2 = sum((s[2] for s in segs), Fraction(0)) / dt
mid = (R + R2) / 2
out = []
for nb, d_b, nt, d_t in segs:
w, w2 = d_b / db, d_t / dt
r = nb / d_b if d_b != 0 else None
r2 = nt / d_t if d_t != 0 else None
r = r2 if r is None else r
r2 = r if r2 is None else r2
mix = (w2 - w) * ((r + r2) / 2 - mid) # type: ignore[operator]
rate = (r2 - r) * (w + w2) / 2 # type: ignore[operator]
out.append((w, w2, r, r2, mix, rate))
return R, R2, out
def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[Segment], by: list[str],
dims: list[str], label: str, num_label: str, den_label: str,
heads: Mapping[str, tuple[Fraction, Any]], parent_row: Mapping[str, Any] | None,
outcome: cmp.Outcome | None, criterion: str) -> Any:
main_share, flat_pct = env.threshold("main_share"), env.threshold("flat_pct")
min_den = env.threshold("min_denominator")
live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0]
dropped = [x for x in segs if x not in live]
R, R2, effects = ratio_effects([(x.base[0], x.base[1], x.target[0], x.target[1]) for x in live])
delta = R2 - R
by_key = {x.key: e for x, e in zip(live, effects)}
gaps: dict[str, Fraction] = {}
for side, total in (("base", R), ("target", R2)):
if side in heads and not closes(total, heads[side][0], heads[side][1]):
gaps[side] = heads[side][0] - total
if parent_row is not None:
sums = {"n_base": sum((x.base[0] for x in live), Fraction(0)), "d_base": sum((x.base[1] for x in live), Fraction(0)),
"n_target": sum((x.target[0] for x in live), Fraction(0)),
"d_target": sum((x.target[1] for x in live), Fraction(0))}
for col, total in sums.items():
expect = to_fraction(parent_row.get(col))
if expect is None or not closes(total, expect, parent_row.get(col)):
gaps.setdefault("base" if col.endswith("base") else "target", (expect or Fraction(0)) - total)
closed = not gaps
mix_total = sum((e[4] for e in effects), Fraction(0))
rate_total = sum((e[5] for e in effects), Fraction(0))
flat = delta == 0 or (R != 0 and abs(delta) / abs(R) < flat_pct)
mix_share = mix_total / delta if delta != 0 else None
rate_share = rate_total / delta if delta != 0 else None
sign = 1 if delta >= 0 else -1
if not closed:
code, subjects = "unclosed", []
elif flat:
code, subjects = "flat", []
elif mix_share >= main_share and rate_share < main_share: # type: ignore[operator]
code = "mix_driven"
subjects = [max(live, key=lambda x: sign * by_key[x.key][4])]
elif rate_share >= main_share and mix_share < main_share: # type: ignore[operator]
code = "rate_driven"
subjects = [max(live, key=lambda x: sign * by_key[x.key][5])]
else:
code, subjects = "both", []
proportion = all(0 <= v[0] <= v[1] for x in live for v in (x.base, x.target))
small = [x for x in live if x.base[1] < min_den or x.target[1] < min_den]
caveats: list[dict[str, str]] = []
eligible = [x for x in live if x not in small]
if delta != 0 and eligible:
weight = sum(((by_key[x.key][0] + by_key[x.key][1]) / 2 for x in eligible), Fraction(0))
against = sum(((by_key[x.key][0] + by_key[x.key][1]) / 2 for x in eligible
if (by_key[x.key][3] - by_key[x.key][2]) * sign < 0), Fraction(0))
if weight > 0 and against / weight >= SIMPSON_SHARE:
star = max(live, key=lambda x: sign * by_key[x.key][4])
caveats.append(ph.caveat("simpson", ph.simpson_text(label, delta > 0, ph.key_text(star.key),
by_key[star.key][0], by_key[star.key][1])))
if small:
caveats.append(ph.caveat("small_denominator", ph.small_denominator_text(
[ph.key_text(x.key) for x in small], den_label, int(env.settings["min_denominator"]))))
nulls = [x for x in segs if "为空" in x.flags]
if nulls:
caveats.append(ph.caveat("null_values", ph.null_segments_text([ph.key_text(x.key) for x in nulls],
ph.say(num_label, "或", den_label))))
if outcome is not None and outcome.verdict != "comparable":
caveats.append(ph.caveat("comparability", outcome.phrase))
total_text = ph.ratio_change(R, R2, proportion=proportion)
if code == "mix_driven":
star = subjects[0]
main = ph.ratio_mix(label, total_text, ph.key_text(star.key), by_key[star.key][0], by_key[star.key][1])
elif code == "rate_driven":
main = ph.ratio_rate(label, total_text, ph.key_text(subjects[0].key))
elif code == "both":
main = ph.ratio_both(label, total_text, mix_share, rate_share)
elif code == "flat":
main = ph.decompose_flat(label, delta / abs(R) if R != 0 else None, None)
else:
main = ph.decompose_unclosed(ph.dims_text(dims), max(gaps.values(), key=abs), ratio=True)
rows = []
for x in sorted(live, key=lambda x: -abs(by_key[x.key][4] + by_key[x.key][5])):
w, w2, r, r2, mix, rate = by_key[x.key]
flags = list(x.flags)
if x.base[1] == 0:
flags.append("基期分母为 0")
if x.target[1] == 0:
flags.append("本期分母为 0")
if x in small:
flags.append("分母小")
rows.append([*x.key, canon(x.base[0]), canon(x.target[0]), canon(x.base[1]), canon(x.target[1]), canon(w),
canon(w2), canon(r), canon(r2), canon(mix), canon(rate), canon(mix + rate),
"、".join(dict.fromkeys(flags)) or None])
for x in dropped:
rows.append([*x.key, canon(x.base[0]), canon(x.target[0]), "0", "0", None, None, None, None, None, None, None,
"两期分母都为 0,不计入"])
table = {"columns": [*by, *_FIXED_RATIO],
"labels": {**dict(zip(by, dims)), "n_base": ph.say("基期", num_label), "n_target": ph.say("本期", num_label),
"d_base": ph.say("基期", den_label), "d_target": ph.say("本期", den_label),
"w_base": "基期权重", "w_target": "本期权重", "r_base": "基期比率", "r_target": "本期比率",
"mix": "结构效应", "rate": "自身效应", "total": "合计影响", "flag": "说明"},
"rows": rows}
values: dict[str, Any] = {
"r_base_total": canon(R), "r_target_total": canon(R2), "delta": canon(delta), "mix_total": canon(mix_total),
"rate_total": canon(rate_total), "mix_share": canon(mix_share), "rate_share": canon(rate_share),
"segments": str(len(live)),
}
if gaps:
values["gap"] = canon(max(gaps.values(), key=abs))
verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects],
"direction": sign_word(delta) if code in ("mix_driven", "rate_driven", "both") else None}
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
phrase=ph.with_caveats(main, caveats), caveats=caveats)
# --------------------------------------------------------------------------
# factor
# --------------------------------------------------------------------------
def _half_ulp_ratio(raw: Any, value: Fraction) -> Fraction:
from app.analysis.common import written_places
return Fraction(1, 2) / Fraction(10) ** written_places(raw) / abs(value) if value != 0 else Fraction(0)
def lmdi(base: Sequence[float], target: Sequence[float], m_base: float, m_target: float) -> list[float]:
"""LMDI-I:contrib_k = L(M', M) · ln(F_k' / F_k),L(a, b) = (a − b) / (ln a − ln b),L(a, a) = a。因子按书写精度取过整、
乘积和乘积列差一点时,各因子按同一比例缩放到乘积恰好等于乘积列(每个因子的对数比各补上 (1/K)·差的对数),Σcontrib 在
浮点误差内等于 M' − M。"""
k = len(base)
fix = (math.log(m_target / math.prod(target)) - math.log(m_base / math.prod(base))) / k
weight = m_base if m_base == m_target else (m_target - m_base) / (math.log(m_target) - math.log(m_base))
return [weight * (math.log(t / b) + fix) for b, t in zip(base, target)]
def factor(raw: Mapping[str, Any], env: Env) -> Any:
used = ["main_share", "flat_pct"]
params, bs, ts, cols, _ = _sides(raw, env, [("product", "乘积列")], by_needed=False)
product = cols["product"]
wanted = as_list(raw.get("factors"))
if not 2 <= len(wanted) <= 4:
raise CallError("bad_factors", "因子列要写 2 到 4 列")
factors = [ts.column(c, "因子列") for c in wanted]
if bs is not ts:
for c in factors:
bs.column(c, "因子列")
if len(set(factors)) != len(factors) or product in factors:
raise CallError("bad_factors", "因子列不能重复,也不能是乘积列本身")
params["factors"] = factors
label = label_for(env, raw.get("label"), ts, product)
if raw.get("label") is not None:
params["label"] = ph.clean_label(raw.get("label")) or None
names = [column_label(env, ts, c) for c in factors]
criterion = ph.criterion_factor(env.settings)
for snap in dict.fromkeys([bs, ts]):
if snap.truncated:
env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [product, *factors])
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行,结果不完整",
criterion=criterion)
if "period" in params:
rows_b = _period_rows(bs, params["period"], params["base"], "基期")
rows_t = _period_rows(ts, params["period"], params["target"], "本期")
else:
rows_b, rows_t = list(range(len(bs.rows))), list(range(len(ts.rows)))
for rows, snap, when in ((rows_b, bs, "基期"), (rows_t, ts, "本期")):
if len(rows) != 1:
raise CallError("duplicate_rows", f"{snap.ref} 里{when}有 {len(rows)} 行:乘积拆解每期要正好一行(在 SQL 里汇总成一行)")
cells: dict[str, list[tuple[Fraction, Any]]] = {}
for when, snap, r in (("base", bs, rows_b[0]), ("target", ts, rows_t[0])):
row = []
for col, what in ((product, "乘积列"), *((c, "因子列") for c in factors)):
value = snap.number(r, col, what)
if value is None:
raise CallError("empty_cell", f"{snap.ref} 第 {r} 行的「{col}」是空的:乘积、因子都要有值")
row.append((value, snap.cell(r, col)))
cells[when] = row
if bs is ts:
env.inputs.use(ts, "data", [params["period"], product, *factors], rows_b + rows_t)
else:
env.inputs.use(bs, "base", [product, *factors], rows_b)
env.inputs.use(ts, "target", [product, *factors], rows_t)
outcome, structure = _comparability(env, params, bs, ts, product)
if structure is not None:
return refuse(env, params, used, "incomparable_sides", f"{structure},不是同一个量的两期,不拆", criterion=criterion)
if any(v <= 0 for row in cells.values() for v, _ in row):
return refuse(env, params, used, "non_positive", "乘积或因子里有不大于 0 的值,对数拆解要求全为正", criterion=criterion)
for when, row in cells.items():
m = row[0][0]
prod = math.prod((v for v, _ in row[1:]), start=Fraction(1))
rel = abs(m - prod) / abs(m)
allowed = max(_FACTOR_REL, sum((_half_ulp_ratio(raw_v, v) for v, raw_v in row[1:]), Fraction(0)))
if rel > allowed:
raise CallError("not_product", ph.say("这几列相乘不等于", column_label(env, ts, product),
f"({'基期' if when == 'base' else '本期'}差 {ph.pct(rel, signed=False)}),"
"不是乘积关系:换成乘起来正好等于它的几列"))
M, M2 = cells["base"][0][0], cells["target"][0][0]
fb = [float(v) for v, _ in cells["base"][1:]]
ft = [float(v) for v, _ in cells["target"][1:]]
contrib = lmdi(fb, ft, float(M), float(M2))
delta = M2 - M
flat = delta == 0 or abs(delta) / abs(M) < env.threshold("flat_pct")
shares = [c / float(delta) for c in contrib] if delta != 0 else [None] * len(contrib)
main_share = float(env.threshold("main_share"))
order = sorted(range(len(factors)), key=lambda i: -abs(contrib[i]))
caveats: list[dict[str, str]] = []
if outcome is not None and outcome.verdict != "comparable":
caveats.append(ph.caveat("comparability", outcome.phrase))
total_text = ph.change(delta, M)
top = max(range(len(factors)), key=lambda i: shares[i] if shares[i] is not None else -math.inf)
if flat:
code, subjects = "flat", []
main = ph.decompose_flat(label, delta / abs(M), None)
elif shares[top] >= main_share: # type: ignore[operator]
code, subjects = "driver", [[names[top]]]
main = ph.factor_driver(label, total_text, names[top], contrib[top], shares[top])
else:
code, subjects = "both", []
main = ph.factor_both(label, total_text, [(names[i], shares[i]) for i in order])
rows = []
for i in order:
b, t = cells["base"][i + 1][0], cells["target"][i + 1][0]
rows.append([names[i], canon(b), canon(t), canon((t - b) / b), canon(contrib[i]),
canon(shares[i]) if shares[i] is not None else None])
table = {"columns": ["factor", "base", "target", "pct", "contrib", "share"],
"labels": {"factor": "因子", "base": "基期", "target": "本期", "pct": "变化率", "contrib": "贡献",
"share": "占变化的比重"},
"rows": rows}
values = {"base_total": canon(M), "target_total": canon(M2), "delta_total": canon(delta),
"pct_total": canon(delta / abs(M)), "top_key": names[top],
"top_share": canon(shares[top]) if shares[top] is not None else None}
verdict = {"code": code, "subjects": subjects, "direction": sign_word(delta) if code != "flat" else None}
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
phrase=ph.with_caveats(main, caveats), caveats=caveats)
__all__ = ["HEADLINE_SHAPE", "PARENT_SHAPE", "SAME_QUERY", "SIMPSON_SHARE", "additive", "factor", "lmdi", "ratio", "ratio_effects"]
+371
View File
@@ -0,0 +1,371 @@
"""差异检验(细案 01 §5):这两组(或两期)真的不一样,还是样本波动。
**参数**:query(一份快照);按组比时 by(分组列,一列);按期比同一组时 period、base、target(各一行,SQL 里先筛好那一组)。
比哪几对:pairs([[A, B], …],A、B 是分组列里的值)、reference(对照组:其余各组各和它比);都不写时两组就比这两组,三组
以上每组和「其余各组合计」比(和含它自己的整体比不独立)。可选 label。
- rate:numerator(成功数,如转化单数)、denominator(试验数,如访客数),每组一行,都要是不小于 0 的整数、成功数不超过
试验数。每组 Wilson 区间,两组之差 Newcombe 区间;最小期望频数不小于 min_expected 用合并比例的 z 检验,否则用 Fisher
精确检验。
- mean:每组一行,mean、sd、n 三列(均值、标准差、样本数),或者 sum、sumsq、n 三列(合计、平方和、样本数:SQLite 没有标准差
函数,SUM(x)、SUM(x*x)、COUNT(*) 都写得出)。Welch t 检验,自由度按 Welch–Satterthwaite。
- rows:value(逐行的值)+ 分组列(或期次列),快照不超过 1000 行、没截断;算子自己算均值、方差,另做敏感性:去掉离均值
最远的那一个值后重算,判定变了加 outlier_sensitive。
比了几对时按 Holm 校正,表里同时给原始 p 和校正后的 p,附 multiple_comparisons。校正后 p < differ_alpha → differs(direction
按 A − B 的正负);否则 no_evidence。用了 Fisher(比率)或有一组样本数小于 min_n(均值)→ small_sample。判定语不出现「p 值」
「显著性」:differs 说「差异站得住」,no_evidence 说「差异可能只是样本波动」,p 值、区间在表和证据面板里。
subjects 是判定说的那一对 [[A], [B]](只比一对时就是那一对,比了几对时是校正后最站得住的那一对;几对都不站得住时为空),
direction 是 A 相对 B。
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from fractions import Fraction
from typing import Any
from app.analysis import phrases as ph
from app.analysis import stats
from app.analysis.common import Env, build, column_label, label_for, need, refuse, sign_word
from app.analysis.contract import CallError
from app.analysis.inputs import Snap, as_list, hashable, same_value
from app.analysis.numbers import canon, to_fraction
TESTS = {"z": "z 检验", "fisher": "Fisher 精确检验", "welch": "Welch t 检验"}
@dataclass(eq=False)
class Group:
key: Any
name: str
rows: list[int] = field(default_factory=list)
x: int = 0 # rate:成功数
n: int = 0 # rate:试验数;mean / rows:样本数
mean: Fraction | None = None
var: Fraction | None = None
values: list[Fraction] = field(default_factory=list)
@dataclass
class Pair:
a: Group
b: Group
test: str | None = None
p: float | None = None
p_adj: float | None = None
diff: Fraction | None = None
ci: tuple[float, float] | None = None
a_value: Fraction | None = None
b_value: Fraction | None = None
def _int(value: Fraction | None, snap: Snap, r: int, column: str) -> int:
if value is None or value.denominator != 1 or value < 0:
raise CallError("not_count", f"{snap.ref} 第 {r} 行的「{column}」不是不小于 0 的整数:比率比较的分子、分母要是计数")
return int(value)
def _pool(name: str, groups: Sequence[Group], mode: str) -> Group:
"""「其余各组合计」:比率把成功数、试验数加起来;均值按各组的均值、方差、样本数合并;逐行的值并在一起。"""
out = Group(("__rest__",), name, rows=[r for g in groups for r in g.rows])
if mode == "rate":
out.x, out.n = sum(g.x for g in groups), sum(g.n for g in groups)
return out
if mode == "rows":
out.values = [v for g in groups for v in g.values]
return _moments(out)
out.n = sum(g.n for g in groups)
if out.n == 0:
return out
out.mean = sum((g.mean * g.n for g in groups if g.mean is not None), Fraction(0)) / out.n
if out.n > 1:
within = sum(((g.n - 1) * g.var for g in groups if g.var is not None and g.n > 1), Fraction(0))
between = sum((g.n * (g.mean - out.mean) ** 2 for g in groups if g.mean is not None), Fraction(0))
out.var = (within + between) / (out.n - 1)
return out
def _moments(group: Group) -> Group:
group.n = len(group.values)
if group.n:
group.mean = sum(group.values, Fraction(0)) / group.n
if group.n > 1:
group.var = sum(((v - group.mean) ** 2 for v in group.values), Fraction(0)) / (group.n - 1) # type: ignore[operator]
return group
def _test(pair: Pair, mode: str, min_expected: Fraction) -> None:
a, b = pair.a, pair.b
if mode == "rate":
if a.n == 0 or b.n == 0:
return
pair.a_value, pair.b_value = Fraction(a.x, a.n), Fraction(b.x, b.n)
pair.diff = pair.a_value - pair.b_value
pair.ci = stats.newcombe(a.x, a.n, b.x, b.n)
if stats.min_expected(a.x, a.n, b.x, b.n) >= min_expected:
pair.test = "z"
pair.p = stats.pooled_z(a.x, a.n, b.x, b.n)[1] # type: ignore[index]
else:
pair.test = "fisher"
pair.p = stats.fisher_exact(a.x, a.n - a.x, b.x, b.n - b.x)
return
if a.mean is None or b.mean is None:
return
pair.a_value, pair.b_value = a.mean, b.mean
pair.diff = a.mean - b.mean
if a.var is None or b.var is None:
return
test = stats.welch(float(a.mean), float(a.var), a.n, float(b.mean), float(b.var), b.n)
if test is None:
return
pair.test, pair.p = "welch", test.p
pair.ci = stats.welch_interval(float(a.mean), float(a.var), a.n, float(b.mean), float(b.var), b.n)
def _verdict(pairs: Sequence[Pair], alpha: Fraction) -> list[Pair]:
"""Holm 校正后站得住的那几对(按校正后的 p 从小到大,一样的按先后)。"""
adjusted = stats.holm([p.p for p in pairs])
for pair, adj in zip(pairs, adjusted):
pair.p_adj = adj
hits = [p for p in pairs if p.p_adj is not None and to_fraction(p.p_adj) < alpha] # type: ignore[operator]
return sorted(hits, key=lambda p: p.p_adj) # type: ignore[arg-type, return-value]
def differ(raw: Mapping[str, Any], env: Env) -> Any:
mode = env.mode
used = ["differ_alpha", "min_expected"] if mode == "rate" else ["differ_alpha", "min_n"]
snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
params: dict[str, Any] = {"query": snap.ref}
by_period = raw.get("period") is not None
if by_period:
group_col = snap.column(raw.get("period"), "期次列")
params.update(period=group_col, base=need(raw, "base", "基期(期次列里的值)"),
target=need(raw, "target", "本期(期次列里的值)"))
else:
wanted = as_list(raw.get("by"))
if len(wanted) != 1:
raise CallError("bad_by", "差异检验按一列分组:分组列写一列;比同一组的两期就写期次列、基期、本期")
group_col = snap.column(wanted[0], "分组列")
params["by"] = [group_col]
if mode == "rate":
cols = {"numerator": snap.column(need(raw, "numerator", "成功数列"), "成功数列"),
"denominator": snap.column(need(raw, "denominator", "试验数列"), "试验数列")}
elif mode == "mean":
if raw.get("mean") is not None:
cols = {"mean": snap.column(raw.get("mean"), "均值列"), "sd": snap.column(need(raw, "sd", "标准差列"), "标准差列"),
"n": snap.column(need(raw, "n", "样本数列"), "样本数列")}
else:
cols = {"sum": snap.column(need(raw, "sum", "合计列(或均值列)"), "合计列"),
"sumsq": snap.column(need(raw, "sumsq", "平方和列"), "平方和列"),
"n": snap.column(need(raw, "n", "样本数列"), "样本数列")}
else:
cols = {"value": snap.column(need(raw, "value", "值列"), "值列")}
params.update(cols)
measure_col = cols.get("numerator") or cols.get("mean") or cols.get("sum") or cols.get("value")
if mode == "rate":
label = ph.pick_label(raw.get("label"), ph.say(column_label(env, snap, cols["numerator"]), "占",
column_label(env, snap, cols["denominator"]), "的比率"))
else:
label = label_for(env, raw.get("label"), snap, measure_col)
if raw.get("label") is not None:
params["label"] = ph.clean_label(raw.get("label")) or None
dim_label = column_label(env, snap, group_col)
criterion = ph.criterion_differ(env.settings, mode) # type: ignore[arg-type]
if snap.truncated:
env.inputs.use(snap, "data", [group_col, *cols.values()])
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行,各组不完整",
criterion=criterion)
groups = _groups(snap, group_col, cols, mode, by_period, params) # type: ignore[arg-type]
pairs = _pairs(raw, params, groups, mode, dim_label, by_period, group_col) # type: ignore[arg-type]
env.inputs.use(snap, "data", [group_col, *cols.values()],
sorted({r for p in pairs for g in (p.a, p.b) for r in g.rows}))
return _decide(env, params, used, pairs, mode, label, criterion) # type: ignore[arg-type]
def _groups(snap: Snap, group_col: str, cols: Mapping[str, str], mode: str, by_period: bool,
params: Mapping[str, Any]) -> dict[Any, Group]:
gi = snap.index(group_col)
groups: dict[Any, Group] = {}
wanted = [params["target"], params["base"]] if by_period else None
for r, row in enumerate(snap.rows):
value = row[gi] if gi < len(row) else None
if wanted is not None:
if not any(same_value(w, value) for w in wanted):
continue
key = hashable(value)
group = groups.setdefault(key, Group(key, ph.key_text([value])))
if mode != "rows" and group.rows:
raise CallError("duplicate_rows", f"{snap.ref} 里「{ph.key_text([value])}」有不止一行:在 SQL 里按 {group_col} "
"汇总,每组一行")
group.rows.append(r)
if mode == "rate":
group.x = _int(snap.number(r, cols["numerator"], "成功数列"), snap, r, cols["numerator"])
group.n = _int(snap.number(r, cols["denominator"], "试验数列"), snap, r, cols["denominator"])
if group.x > group.n:
raise CallError("not_count", f"{snap.ref} 第 {r} 行的成功数大于试验数:分子、分母要是同一批里的「成功几个、"
"一共几个」")
elif mode == "mean":
n = _int(snap.number(r, cols["n"], "样本数列"), snap, r, cols["n"])
group.n = n
if "mean" in cols:
group.mean = snap.number(r, cols["mean"], "均值列")
sd = snap.number(r, cols["sd"], "标准差列")
group.var = sd * sd if sd is not None and n > 1 else None
else:
total = snap.number(r, cols["sum"], "合计列")
sq = snap.number(r, cols["sumsq"], "平方和列")
if total is not None and n > 0:
group.mean = total / n
if sq is not None and n > 1:
group.var = max(Fraction(0), (sq - total * total / n) / (n - 1))
else:
v = snap.number(r, cols["value"], "值列")
if v is not None:
group.values.append(v)
if mode == "rows":
for g in groups.values():
_moments(g)
if by_period:
for w, what in ((params["target"], "本期"), (params["base"], "基期")):
if not any(same_value(w, g.key) for g in groups.values()):
raise CallError("period_not_found", f"期次列「{group_col}」里没有「{w}」:{what}要写 {snap.ref} 里有的期次值")
return groups
def _find(groups: Mapping[Any, Group], value: Any, column: str) -> Group:
found = next((g for g in groups.values() if same_value(value, g.key)), None)
if found is None:
shown = "、".join(g.name for g in list(groups.values())[:8])
raise CallError("unknown_group", f"「{value}」不是「{column}」里的值:要比的组写 {shown} 里的")
return found
def _pairs(raw: Mapping[str, Any], params: dict[str, Any], groups: dict[Any, Group], mode: str, dim_label: str,
by_period: bool, group_col: str) -> list[Pair]:
if by_period:
target = _find(groups, params["target"], group_col)
base = _find(groups, params["base"], group_col)
return [Pair(target, base)]
if raw.get("pairs") is not None:
spec = raw.get("pairs")
if not isinstance(spec, list) or not spec or not all(isinstance(p, (list, tuple)) and len(p) == 2 for p in spec):
raise CallError("bad_pairs", "要比的几对写成 [[A, B], …],A、B 是分组列里的值")
out = []
for a, b in spec:
ga, gb = _find(groups, a, group_col), _find(groups, b, group_col)
if ga is gb:
raise CallError("bad_pairs", f"「{ga.name}」不能和自己比")
out.append(Pair(ga, gb))
params["pairs"] = [[p.a.key, p.b.key] for p in out]
return out
if len(groups) < 2:
raise CallError("too_few_groups", f"「{group_col}」里只有 {len(groups)} 组,至少要两组才比得了")
if raw.get("reference") is not None:
ref = _find(groups, raw.get("reference"), group_col)
params["reference"] = ref.key
return [Pair(g, ref) for g in groups.values() if g is not ref]
items = list(groups.values())
if len(items) == 2:
return [Pair(items[0], items[1])]
rest_name = ph.say("其余各", dim_label)
return [Pair(g, _pool(rest_name, [o for o in items if o is not g], mode)) for g in items]
def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair], mode: str, label: str,
criterion: str) -> Any:
alpha = env.threshold("differ_alpha")
min_expected = env.threshold("min_expected") if mode == "rate" else Fraction(0)
for pair in pairs:
_test(pair, mode, min_expected)
hits = _verdict(pairs, alpha)
caveats: list[dict[str, str]] = []
if mode == "rows":
flipped = _sensitivity(pairs, mode, alpha, min_expected, bool(hits))
if flipped is not None:
caveats.append(ph.caveat("outlier_sensitive", ph.outlier_text(*flipped)))
hits = _verdict(pairs, alpha)
if mode == "rate" and any(p.test == "fisher" for p in pairs):
caveats.append(ph.caveat("small_sample", ph.small_sample_rate_text(int(env.settings["min_expected"]))))
if mode != "rate":
min_n = int(env.settings["min_n"])
small = sorted((g for p in pairs for g in (p.a, p.b) if g.n < min_n), key=lambda g: g.n)
if small:
caveats.append(ph.caveat("small_sample", ph.small_sample_mean_text(small[0].name, small[0].n, min_n)))
tested = [p for p in pairs if p.p is not None]
if len(tested) > 1:
caveats.append(ph.caveat("multiple_comparisons", ph.multiple_text(len(tested))))
fmt = (lambda v: ph.pct(v, signed=False)) if mode == "rate" else (lambda v: ph.amount(v))
if hits:
code = "differs"
head = hits[0]
main = ph.differ_phrase([ph.differ_pair(p.a.name, p.b.name, label, fmt(p.a_value), fmt(p.b_value),
p.diff > 0) for p in hits], # type: ignore[operator]
len(pairs))
subjects = [[head.a.name], [head.b.name]]
direction = sign_word(head.diff) # type: ignore[arg-type]
else:
code = "no_evidence"
head = pairs[0] if len(pairs) == 1 else None
sizes = (ph.amount(head.a.n), ph.amount(head.b.n)) if head is not None else None
main = ph.no_evidence_phrase(head.a.name if head else None, head.b.name if head else None, label, sizes,
len(pairs))
subjects = [[head.a.name], [head.b.name]] if head is not None else []
direction = None
value_name = "比率" if mode == "rate" else "均值"
rows = []
for p in pairs:
flags = []
if p.p is None:
flags.append("样本不够,算不了")
if p.test == "fisher":
flags.append("样本少,用了精确检验")
rows.append([p.a.name, p.b.name, canon(p.a_value), canon(p.b_value), canon(p.diff),
canon(p.ci[0]) if p.ci else None, canon(p.ci[1]) if p.ci else None, str(p.a.n), str(p.b.n),
TESTS.get(p.test or "") or None, canon(p.p), canon(p.p_adj), "、".join(flags) or None])
table = {"columns": ["a", "b", "a_value", "b_value", "diff", "ci_lo", "ci_hi", "a_n", "b_n", "test", "p", "p_adj",
"flag"],
"labels": {"a": "组", "b": "对照组", "a_value": f"组的{value_name}", "b_value": f"对照组的{value_name}",
"diff": "差", "ci_lo": "差的区间下限", "ci_hi": "差的区间上限", "a_n": "组的样本量",
"b_n": "对照组的样本量", "test": "检验", "p": "p 值", "p_adj": "校正后的 p 值", "flag": "说明"},
"rows": rows}
shown = head if head is not None else pairs[0]
values = {"a": canon(shown.a_value), "b": canon(shown.b_value), "diff": canon(shown.diff),
"ci_lo": canon(shown.ci[0]) if shown.ci else None, "ci_hi": canon(shown.ci[1]) if shown.ci else None,
"p": canon(shown.p), "p_adj": canon(shown.p_adj), "n_a": str(shown.a.n), "n_b": str(shown.b.n),
"test": shown.test, "pairs": str(len(pairs)), "differs": str(len(hits))}
verdict = {"code": code, "subjects": subjects, "direction": direction}
phrase = ph.with_caveats(main, caveats, skip=("multiple_comparisons",) if len(pairs) > 1 else ())
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, phrase=phrase,
caveats=caveats)
def _sensitivity(pairs: list[Pair], mode: str, alpha: Fraction, min_expected: Fraction,
differs: bool) -> tuple[bool, bool] | None:
"""rows 模式的敏感性:去掉比较的各组里离均值最远的那一个值重算,判定变了交回 (去掉的是不是偏大的那一笔, 现在站不站得住)。"""
everything = [(g, i, v) for g in dict.fromkeys(g for p in pairs for g in (p.a, p.b)) for i, v in enumerate(g.values)]
if len(everything) < 3:
return None
mean = sum((v for _, _, v in everything), Fraction(0)) / len(everything)
group, index, value = max(everything, key=lambda t: abs(t[2] - mean))
trimmed = []
for p in pairs:
def cut(g: Group) -> Group:
if g is not group and not (g.key == ("__rest__",) and any(v is value for v in g.values)):
return g
values = list(g.values)
for k, v in enumerate(values):
if v is value:
del values[k]
break
return _moments(Group(g.key, g.name, rows=g.rows, values=values))
trimmed.append(Pair(cut(p.a), cut(p.b)))
for p in trimmed:
_test(p, mode, min_expected)
now = bool(_verdict(trimmed, alpha))
_verdict(pairs, alpha)
return (value > mean, now) if now != differs else None
__all__ = ["TESTS", "differ"]
+218
View File
@@ -0,0 +1,218 @@
"""目录事实:结果列给人看的叫法、度量类型,表的业务日期(细案 01 §2.1、§6、§7.3)。
算子要知道「gmv 是流量还是比率」(比率当可加是调用错误;窗口不等时流量判不可比、其余只提醒)、「region 叫地区」(判定语
写「分散在各地区」)、「orders 的业务日期是 order_date」(可比性检查按它取时间条件)。这些在数据目录里,可目录描述的是表的
列,快照里的是 SQL 的结果列,所以先按 SQL 认结果列取自哪张表的哪一列,认法同单元格的显示规则(engine/cell_format:最外层
SELECT 里按别名或列名找到那一项,看它是不是聚合、聚合的参数是不是一列、这一列在 FROM / JOIN 的哪张实表上),再查那一列
的目录项:
- SUM(x)、不聚合的 x:x 的度量类型;COUNT(*) / COUNT(x):计数,按流量算;AVG(x):平均值,按比率算(不能按段相加);
COUNT(DISTINCT x)、MIN / MAX、算式:认不出(当没标)。
- 叫法:x 的中文名(目录里的 label);没有就是没有,调用方用列名。
- 只认没被驳回的项(catalog.visible_notes)。度量类型另记有没有人确认过(confirmed / verified):只有确认过的才拿来判
调用错误,推断的只写进附带说明(同 SQL 检查「宁可漏报,不可误报」)。
**复算不读目录**:run 时查过的都记进结论的 params["catalog"](record),复算时 Facts 从那里读,同一个键得到同一个答案;
目录事后改了,封存的结论照样复算得出来。
"""
from __future__ import annotations
from collections.abc import Mapping
from functools import lru_cache
from typing import Any
from app.data.names import name_key
#: 有确证的项状态:只有它们拿来判调用错误
_STRONG = frozenset({"confirmed", "verified"})
def _notes_of(entry: Any) -> Mapping[str, Any] | None:
"""目录里一张表的 notes:CatalogEntry(有 .notes)或 notes 本身;去掉被驳回的项。"""
from app.data import catalog
notes = getattr(entry, "notes", entry)
return catalog.visible_notes(notes) if isinstance(notes, Mapping) else None
def _table_notes(catalog: Mapping[str, Any] | None, table: str | None) -> Mapping[str, Any] | None:
if not catalog or not table:
return None
wanted = {name_key(table), name_key(table.rsplit(".", 1)[-1])}
for name, entry in catalog.items():
if name_key(str(name)) in wanted or name_key(str(name).rsplit(".", 1)[-1]) in wanted:
return _notes_of(entry)
return None
def _item(notes: Mapping[str, Any] | None, column: str, key: str) -> Mapping[str, Any] | None:
columns = (notes or {}).get("columns") if isinstance(notes, Mapping) else None
if not isinstance(columns, Mapping):
return None
for name, items in columns.items():
if name_key(str(name)) == name_key(column) and isinstance(items, Mapping):
item = items.get(key)
return item if isinstance(item, Mapping) else None
return None
@lru_cache(maxsize=128)
def _outer_select(sql: str) -> Any:
"""最外层的 SELECT(engine/cell_format 的读法);同一条 SQL 只解析一次(只读,不改这棵树)。"""
from app.engine import cell_format as cf
try:
return cf._outer_select(sql)
except Exception: # noqa: BLE001
return None
def source_of(sql: str, column: str) -> tuple[str | None, str | None, str | None, bool] | None:
"""结果列 column 取自哪里:(表, 列, 聚合, 去重)。聚合是 sum / count / avg / min / max / other(别的聚合)或 None(不聚合);
表在 SELECT * 、子查询、认不准的时候为 None。SQL 解析不了、结果列在最外层 SELECT 里找不到为 None。"""
from sqlglot import exp
from app.engine import cell_format as cf
try:
select = _outer_select(sql)
if select is None:
return None
item = cf._projection(select, column)
if item is None:
return None
value = cf._unwrap(item.unalias())
agg: str | None = None
distinct = False
if isinstance(value, exp.AggFunc):
agg = {exp.Sum: "sum", exp.Count: "count", exp.Avg: "avg", exp.Min: "min", exp.Max: "max"}.get(
type(value), "other")
arg = value.this
if isinstance(arg, exp.Distinct):
distinct = True
arg = arg.expressions[0] if len(arg.expressions) == 1 else None
value = cf._unwrap(arg) if arg is not None else None
if not isinstance(value, exp.Column) or isinstance(value.this, exp.Star):
return None, None, agg, distinct
tables = cf._tables(select)
table = tables.get(name_key(value.table)) if value.table else \
(next(iter(tables.values())) if len(tables) == 1 else None)
return table, value.name, agg, distinct
except Exception: # noqa: BLE001 - 认不出就当没标
return None
def date_source(sql: str, column: str) -> tuple[str, str] | None:
"""期次结果列取自哪张表的哪个日期列:那一项选取里只引用了一列(order_date、DATE(order_date)、
strftime('%Y-%m', order_date))时就是它。认不出为 None。"""
from sqlglot import exp
from app.engine import cell_format as cf
try:
select = _outer_select(sql)
item = cf._projection(select, column) if select is not None else None
if item is None:
return None
cols = [c for c in item.unalias().find_all(exp.Column) if not isinstance(c.this, exp.Star)]
if len({(name_key(c.table), name_key(c.name)) for c in cols}) != 1:
return None
ref = cols[0]
tables = cf._tables(select)
table = tables.get(name_key(ref.table)) if ref.table else \
(next(iter(tables.values())) if len(tables) == 1 else None)
return (table, ref.name) if table else None
except Exception: # noqa: BLE001
return None
class Facts:
"""目录事实的取法:run 时 catalogs 是 context["catalogs"](数据源名 → {表名: notes 或 CatalogEntry}),复算时 recorded 是
结论里记下的 params["catalog"]。查过的都记下(record),同一个键只认一次。"""
def __init__(self, catalogs: Mapping[str, Any] | None = None, recorded: Mapping[str, Any] | None = None) -> None:
self._catalogs = catalogs or {}
self._recorded = recorded
self._labels: dict[str, str | None] = {}
self._measures: dict[str, tuple[str | None, bool]] = {}
self._dates: dict[str, str | None] = {}
def _catalog(self, source: str) -> Mapping[str, Any] | None:
found = self._catalogs.get(source)
return found if isinstance(found, Mapping) else None
def label(self, snap: Any, column: str) -> str | None:
"""结果列给人看的叫法(目录里取自的那一列的中文名);没有为 None。"""
if column in self._labels:
return self._labels[column]
if self._recorded is not None:
value = (self._recorded.get("labels") or {}).get(column)
else:
value = None
src = source_of(snap.sql, column)
if src is not None and src[1] is not None:
item = _item(_table_notes(self._catalog(snap.source), src[0]), src[1], "label")
text = item.get("value") if item else None
value = text.strip() if isinstance(text, str) and text.strip() else None
self._labels[column] = value
return value
def measure(self, snap: Any, column: str) -> tuple[str | None, bool]:
"""结果列的度量类型和有没有人确认过:(flow / stock / ratio / identifier / status / attribute 或 None, 确认过)。"""
if column in self._measures:
return self._measures[column]
if self._recorded is not None:
kind = (self._recorded.get("measures") or {}).get(column)
out = (kind, kind is not None and column not in (self._recorded.get("unconfirmed") or []))
else:
out = self._live_measure(snap, column)
self._measures[column] = out
return out
def _live_measure(self, snap: Any, column: str) -> tuple[str | None, bool]:
src = source_of(snap.sql, column)
if src is None:
return None, False
table, name, agg, distinct = src
if agg == "count" and not distinct:
return "flow", True
if agg == "avg":
return "ratio", True
if agg not in (None, "sum") or name is None:
return None, False
item = _item(_table_notes(self._catalog(snap.source), table), name, "measure")
kind = item.get("value") if item else None
if not isinstance(kind, str):
return None, False
return kind, item.get("status") in _STRONG
def business_date(self, source: str, table: str) -> str | None:
"""表的业务日期列(目录的 business_date);没写为 None。"""
key = name_key(table)
if key in self._dates:
return self._dates[key]
if self._recorded is not None:
value = (self._recorded.get("dates") or {}).get(key)
else:
notes = _table_notes(self._catalog(source), table)
item = notes.get("business_date") if isinstance(notes, Mapping) else None
raw = item.get("value") if isinstance(item, Mapping) else None
col = raw.get("column") if isinstance(raw, Mapping) else None
value = col if isinstance(col, str) and col.strip() else None
self._dates[key] = value
return value
def record(self) -> dict[str, Any] | None:
"""查过、查得到的事实(写进 params["catalog"]);一项都没有为 None。"""
out: dict[str, Any] = {}
labels = {k: v for k, v in self._labels.items() if v is not None}
measures = {k: v[0] for k, v in self._measures.items() if v[0] is not None}
unconfirmed = sorted(k for k, v in self._measures.items() if v[0] is not None and not v[1])
dates = {k: v for k, v in self._dates.items() if v is not None}
for key, value in (("labels", labels), ("measures", measures), ("unconfirmed", unconfirmed), ("dates", dates)):
if value:
out[key] = value
return out or None
__all__ = ["Facts", "date_source", "source_of"]
+222
View File
@@ -0,0 +1,222 @@
"""算子的输入:按名字取快照、拆单元格的写法、按列名认列、遮罩列、度量格换成精确的数、同一键同一期多行(细案 01 §1、§8.1)。
- 名字:agent 写本节点查过的编号(Q3,和交回模型的「【证据 Q3】」同一个),分析卡写来源节点 id;resolve 取不到就是调用错误。
- 单元格写成「名字.r行.列」(Q1.r0.gmv),行从 0 数;名字里可以有点(取最后一个「.r数字.」切开)。
- 列名比较不分大小写,按快照的 columns 定位:一字不差的优先;只差大小写的有好几列时是调用错误。
- 遮罩列(快照的 masked,不分大小写)不能当维度、度量、期次:调用错误。结论里只出现非遮罩列的格。
- 度量格:数(含 DECIMAL 落成的数字文本)换成 Fraction;空值为 None(各算子按细案 §1 处理);文本、布尔是调用错误。
查询时记了类型的列按类型认:text 列里的 "2026" 不当数。
调用错误(contract.CallError)的说法一种,写给 agent:说清哪里不对、怎么改。
"""
from __future__ import annotations
import math
import re
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, field
from fractions import Fraction
from typing import Any
from app.analysis.contract import AnalysisInput, CallError, InputSnapshot
from app.analysis.numbers import canon, to_fraction
#: 单元格的写法:名字.r行.列
_CELL = re.compile(r"^(?P<name>.+)\.r(?P<row>\d+)\.(?P<column>.+)$")
#: 报错里列出的列最多几个
_LIST_MAX = 12
def _listed(columns: Sequence[str]) -> str:
shown = "、".join(columns[:_LIST_MAX])
return shown + (f"等 {len(columns)} 列" if len(columns) > _LIST_MAX else "")
def hashable(value: Any) -> Any:
"""维度值当键:文本、数、布尔、空值原样;别的(列表、对象)和非数、无穷写成文字(同 engine/compare._hashable)。"""
if isinstance(value, float) and not math.isfinite(value):
return repr(value)
return value if isinstance(value, (str, int, float, bool)) or value is None else repr(value)
def same_value(a: Any, b: Any) -> bool:
"""调用里写的期次、段和快照里的值是不是同一个:相等,或者写出来一样("2026" 和 2026、" 华东" 和 "华东")。"""
if a == b and type(a) is type(b):
return True
if isinstance(a, bool) or isinstance(b, bool):
return a == b
fa, fb = to_fraction(a), to_fraction(b)
if fa is not None and fb is not None and not isinstance(a, str) and not isinstance(b, str):
return fa == fb
return str(a).strip() == str(b).strip()
@dataclass(eq=False)
class Snap:
"""一份输入快照(InputSnapshot 的薄包装)。"""
ref: str
data: Mapping[str, Any]
dialect: str | None = None
columns: list[str] = field(default_factory=list)
rows: list[list[Any]] = field(default_factory=list)
def __post_init__(self) -> None:
self.columns = [str(c) for c in self.data.get("columns") or []]
self.rows = [list(r) if isinstance(r, (list, tuple)) else [] for r in self.data.get("rows") or []]
@property
def artifact(self) -> str | None:
value = self.data.get("artifact")
return value if isinstance(value, str) and value else None
@property
def source(self) -> str:
return str(self.data.get("source") or "")
@property
def sql(self) -> str:
return str(self.data.get("sql") or "")
@property
def truncated(self) -> bool:
return bool(self.data.get("truncated"))
@property
def masked(self) -> set[str]:
"""不能当维度、度量的列(小写):入口交来的 masked;没交(复算时取回的是快照工件本身)就按快照记下的遮罩列
(数据源设的 mask_columns、查询时整列遮掉的 masked_at_query)。"""
found = self.data.get("masked")
if found is None:
found = [*(self.data.get("mask_columns") or []), *(self.data.get("masked_at_query") or [])]
return {str(m).strip().lower() for m in found or [] if str(m).strip()}
def kind(self, column: str) -> str | None:
"""查询时记下的这一列的类型(column_types 是 {列: 类型} 或按列序的列表)。"""
types = self.data.get("column_types")
if isinstance(types, Mapping):
kind = types.get(column)
elif isinstance(types, list) and column in self.columns:
i = self.columns.index(column)
kind = types[i] if i < len(types) else None
else:
kind = None
return kind if isinstance(kind, str) else None
def column(self, wanted: Any, role: str) -> str:
"""按名字认一列(不分大小写),认出的是快照里的写法。不存在、只差大小写的好几列、遮罩列都是调用错误。"""
if not isinstance(wanted, str) or not wanted.strip():
raise CallError("missing_column", f"没写{role}:写 {self.ref} 里的列名")
name = wanted.strip()
hits = [c for c in self.columns if c == name] or [c for c in self.columns if c.lower() == name.lower()]
if not hits:
raise CallError("unknown_column", f"{self.ref} 里没有「{name}」这一列,{role}要写它的列名:{self.ref} 的列有 "
f"{_listed(self.columns)}")
if len(hits) > 1:
raise CallError("ambiguous_column", f"{self.ref} 里有好几列都叫「{name}」(只差大小写):{role}写出确切的列名")
column = hits[0]
if column.lower() in self.masked:
raise CallError("masked_column", f"「{column}」是遮罩列,不能当{role}:换一列,或者在 SQL 里不选它")
return column
def index(self, column: str) -> int:
return self.columns.index(column)
def cell(self, row: int, column: str) -> Any:
r = self.rows[row]
i = self.index(column)
return r[i] if i < len(r) else None
def number(self, row: int, column: str, role: str) -> Fraction | None:
"""一格换成精确的数;空值为 None。不是数是调用错误。"""
raw = self.cell(row, column)
if raw is None:
return None
if isinstance(raw, bool) or (isinstance(raw, str) and self.kind(column) == "text"):
raise self._not_numeric(row, column, raw, role)
if isinstance(raw, str):
from app.engine.expressions import numeric_text
if numeric_text(raw.strip()) is None:
raise self._not_numeric(row, column, raw, role)
value = to_fraction(raw)
if value is None:
raise self._not_numeric(row, column, raw, role)
return value
def _not_numeric(self, row: int, column: str, raw: Any, role: str) -> CallError:
shown = str(raw).replace("\n", " ")[:20]
return CallError("not_numeric", f"{self.ref} 的「{column}」不是数值列(第 {row} 行是「{shown}」):{role}要选数值列")
class Inputs:
"""按名字取输入,记下结论的 inputs(同一份快照、同一格只记一次,按用到的先后)。"""
def __init__(self, resolve: Callable[[str], InputSnapshot | None],
dialect: Callable[[str, Mapping[str, Any]], str | None] | None = None) -> None:
self._resolve = resolve
self._dialect = dialect
self._snaps: dict[str, Snap] = {}
self.records: list[AnalysisInput] = []
def snap(self, name: Any, role: str) -> Snap:
"""按名字取一份快照。"""
if not isinstance(name, str) or not name.strip():
raise CallError("missing_query", f"没写{role}:写本节点查过的查询编号(如 Q3)")
key = name.strip()
if key not in self._snaps:
found = self._resolve(key)
if not isinstance(found, Mapping):
raise CallError("unknown_query", f"没有叫「{key}」的查询结果:{role}写本节点查过的查询编号(如 Q3)")
self._snaps[key] = Snap(key, found, self._dialect(key, found) if self._dialect else found.get("dialect"))
return self._snaps[key]
def cell(self, ref: Any, role: str, label: str) -> tuple[Snap, int, str, Fraction | None]:
"""「名字.r行.列」→ (快照, 行, 列, 值)。"""
found = _CELL.match(ref.strip()) if isinstance(ref, str) else None
if found is None:
raise CallError("bad_cell", f"{label}要写成单元格「编号.r行.列」(如 Q1.r0.gmv),现在是「{ref}」")
snap = self.snap(found.group("name"), label)
row = int(found.group("row"))
if row >= len(snap.rows):
raise CallError("row_out_of_range", f"{snap.ref} 没有第 {row} 行(共 {len(snap.rows)} 行,从 r0 数起):改{label}")
column = snap.column(found.group("column"), label)
value = snap.number(row, column, label)
self.record({"ref": f"{snap.ref}.r{row}.{column}", "role": role, "row": row, "column": column,
"value": canon(value)}, snap)
return snap, row, column, value
def use(self, snap: Snap, role: str, columns: Sequence[str], rows: Sequence[int] | None = None) -> None:
"""记下一份快照的用法:用了哪几列、哪几行(全用了写 all)。"""
used = sorted(set(rows)) if rows is not None else None
shown: Any = "all" if used is None or used == list(range(len(snap.rows))) else used
self.record({"ref": snap.ref, "role": role, "columns": list(dict.fromkeys(columns)), "rows": shown}, snap)
def record(self, entry: dict[str, Any], snap: Snap) -> None:
out: dict[str, Any] = {"ref": entry["ref"]}
if snap.artifact:
out["artifact"] = snap.artifact
if snap.dialect:
out["dialect"] = snap.dialect
if snap.masked:
out["masked"] = sorted(snap.masked) # 只记列名(复算按它认遮罩列,快照工件里不一定都有)
out.update({k: v for k, v in entry.items() if k != "ref"})
if out not in self.records:
self.records.append(out) # type: ignore[arg-type]
def parent(self, name: str, artifact: str) -> None:
entry: dict[str, Any] = {"ref": name, "artifact": artifact, "role": "parent"}
if entry not in self.records:
self.records.append(entry) # type: ignore[arg-type]
def as_list(value: Any) -> list[Any]:
"""参数里一个值或一组值:都规范成列表(None 是空列表)。"""
if value is None:
return []
if isinstance(value, (list, tuple)):
return list(value)
return [value]
__all__ = ["Inputs", "Snap", "as_list", "hashable", "same_value"]
+143
View File
@@ -0,0 +1,143 @@
"""分析结论里的数:精确的有理数进、规范的十进制字符串出(细案 01 §1「数用十进制」)。
结论要能逐字节复算(证据面板、导出、可验证报告的复算和封存的逐字段比,评测作答前后各算一次逐字比),所以:
- **算**:快照里的数一律换成 Fraction(浮点按最短写法 repr,数字文本按字面,DECIMAL 落成的文本不经浮点),加减乘除全程
精确——各段之和逐位等于总变化,Σ结构效应 + Σ自身效应逐位等于比率的变化。只有要开方、取对数、查分布的统计量是浮点。
- **写**:canon 把数写成规范的十进制字符串:不带指数、不带正号、去掉多余的 0、没有「-0」。能精确写成有限小数的(快照里的
数和它们的和差,0.25 这样的商)照写;除不尽的和浮点按 SIG 位有效数字四舍五入(ROUND_HALF_UP)。浮点只留 12 位有效
数字:不同机器的数学库在最后一两位上可能不一样,留 12 位同一份输入在哪台机器上复算都是同一串字。
"""
from __future__ import annotations
import math
from decimal import Decimal, InvalidOperation
from fractions import Fraction
from typing import Any
#: 除不尽的数、浮点保留几位有效数字
SIG = 12
#: 能精确写成有限小数、但有效数字超过这么多位的,也按 SIG 位写(只会出现在反复除以 2 这类极端情形)
_EXACT_MAX_DIGITS = 30
def to_fraction(value: Any) -> Fraction | None:
"""一个数换成精确的 Fraction:整数、Decimal、Fraction 原样;浮点按最短写法(repr);数字文本按字面。布尔、非数、无穷、
认不出的为 None。"""
if isinstance(value, bool) or value is None:
return None
if isinstance(value, Fraction):
return value
if isinstance(value, int):
return Fraction(value)
if isinstance(value, float):
if not math.isfinite(value):
return None
return Fraction(Decimal(repr(value)))
if isinstance(value, Decimal):
return Fraction(value) if value.is_finite() else None
if isinstance(value, str):
try:
d = Decimal(value.strip())
except (InvalidOperation, ValueError):
return None
return Fraction(d) if d.is_finite() else None
return None
def _terminating(q: Fraction) -> int | None:
"""q 能精确写成有限小数时要几位小数;不能为 None。"""
d = q.denominator
twos = fives = 0
while d % 2 == 0:
d //= 2
twos += 1
while d % 5 == 0:
d //= 5
fives += 1
return max(twos, fives) if d == 1 else None
def _plain(digits: int, exp10: int) -> str:
"""digits × 10^exp10(digits ≥ 0)写成不带指数、去尾零的十进制。"""
text = str(digits)
if exp10 >= 0:
return text + "0" * exp10 if digits else "0"
places = -exp10
text = text.rjust(places + 1, "0")
head, tail = text[:-places], text[-places:].rstrip("0")
return f"{head}.{tail}" if tail else head
def _decade(a: Fraction) -> int:
"""a > 0 的十进制量级:10^e ≤ a < 10^(e+1) 的 e。"""
try:
e = math.floor(math.log10(a.numerator) - math.log10(a.denominator))
except (ValueError, OverflowError):
e = len(str(a.numerator)) - len(str(a.denominator))
while Fraction(10) ** e > a:
e -= 1
while Fraction(10) ** (e + 1) <= a:
e += 1
return e
def round_sig(q: Fraction, sig: int = SIG) -> Fraction:
"""按 sig 位有效数字四舍五入(ROUND_HALF_UP)。"""
if q == 0:
return Fraction(0)
sign = -1 if q < 0 else 1
a = abs(q)
e = _decade(a)
scale = Fraction(10) ** (sig - 1 - e)
m = math.floor(a * scale + Fraction(1, 2))
return sign * Fraction(m) / scale
def round_places(q: Fraction, places: int) -> Fraction:
"""按 places 位小数四舍五入(ROUND_HALF_UP,负数按绝对值舍入,同 Decimal 的 ROUND_HALF_UP)。"""
sign = -1 if q < 0 else 1
scale = Fraction(10) ** places
return sign * Fraction(math.floor(abs(q) * scale + Fraction(1, 2))) / scale
def _write(q: Fraction) -> str:
"""有限小数的 q 写出来(调用方保证 q 能精确写成有限小数)。"""
places = _terminating(q) or 0
sign = "-" if q < 0 else ""
digits = abs(q) * (10 ** places)
assert digits.denominator == 1
return sign + _plain(int(digits), -places)
def canon(value: Any) -> str | None:
"""规范的十进制字符串(见文件头)。None、非数、无穷为 None。"""
if isinstance(value, float) and math.isfinite(value):
q = round_sig(to_fraction(value) or Fraction(0))
else:
q = to_fraction(value)
if q is None:
return None
places = _terminating(q)
if places is None or len(str(int(abs(q) * 10 ** places)).rstrip("0")) > _EXACT_MAX_DIGITS:
q = round_sig(q)
if q == 0:
return "0"
return _write(q)
def fixed(q: Fraction, places: int, *, strip: bool = True, group: bool = False) -> str:
"""按 places 位小数四舍五入写出来;strip 去掉尾零,group 加千分位。没有正号;舍成 0 的写「0」(不写「-0」)。"""
r = round_places(q, places)
sign = "-" if r < 0 else ""
scaled = abs(r) * (10 ** places)
whole, frac = divmod(int(scaled), 10 ** places)
head = f"{whole:,}" if group else str(whole)
tail = str(frac).rjust(places, "0") if places > 0 else ""
if strip:
tail = tail.rstrip("0")
text = f"{head}.{tail}" if tail else head
return sign + text
__all__ = ["SIG", "canon", "fixed", "round_places", "round_sig", "to_fraction"]
+224
View File
@@ -0,0 +1,224 @@
"""期次:快照里期次列的值认成可以排序、能判断连不连续、知道覆盖哪几天的期(细案 01 §3「期次认得出、能排序」)。
认这几种写法(一列里要是同一种):
- 日期:DATE / DATETIME(零点)、'YYYY-MM-DD'、'YYYYMMDD'、8 位整数。粒度不写时按值推断:都是某月 1 日(或都是月末)
且相邻两期隔整月 → month(月份都在 1、4、7、10 月且隔整季 → quarter);相邻两期都隔 7 天的整数倍 → week(这个日期当
一周的第一天);否则 day。
- 月份:'YYYY-MM'、'YYYYMM'(6 位,月份 01–12)→ month。
- ISO 周:'YYYY-Www'(2026-W42 是 2026-10-12 到 10-18)→ week。SQLite 的 strftime('%W') 不是 ISO 周号(年初那几天
算第 0 周),按它写的期次覆盖的日子会差一周:方法里要求期次写成每周第一天的日期。
- 季度:'YYYY-Qn'、'YYYYQn' → quarter。
- 整数序号(不是上面几种的整数)→ seq:能排序、能判连续,不知道覆盖哪几天(本期完不完整核对不了)。
带时刻(非零点)的、写法混着的、认不出的都算认不出(period_unreadable)。
每一期记 ordinal:同一粒度里相邻两期的 ordinal 差 1(日:天数,周:从第一期起第几个 7 天,月:年 × 12 + 月,季:年 × 4 +
季,序号:本身),据此排序、判缺期、按 lag 找比较期。start / end 是这一期覆盖的第一天和最后一天(含),seq 没有。
"""
from __future__ import annotations
import calendar
import re
from collections.abc import Sequence
from dataclasses import dataclass
from datetime import date, datetime, timedelta
from typing import Any
from app.data import coverage
GRAINS = ("day", "week", "month", "quarter")
#: 粒度的叫法:数量词(过去 12 周、12 个月)、每…(每周、每月)
COUNT_WORD = {"day": "天", "week": "周", "month": "个月", "quarter": "个季度", "seq": "期"}
PER_WORD = {"day": "天", "week": "周", "month": "月", "quarter": "季度", "seq": "期"}
#: 粒度不写时各粒度的缺省 lag:日数据和上周同一天比(避开星期效应),其余和上一期比
DEFAULT_LAG = {"day": 7, "week": 1, "month": 1, "quarter": 1, "seq": 1}
_MONTH = re.compile(r"^(\d{4})-?(\d{2})$")
_WEEK = re.compile(r"^(\d{4})-?W(\d{1,2})$", re.I)
_QUARTER = re.compile(r"^(\d{4})-?Q([1-4])$", re.I)
@dataclass(frozen=True)
class Period:
raw: Any # 快照里的原值
ordinal: int
start: date | None
end: date | None
def _month_end(d: date) -> date:
return d.replace(day=calendar.monthrange(d.year, d.month)[1])
def _quarter_start(d: date) -> date:
return date(d.year, 3 * ((d.month - 1) // 3) + 1, 1)
def _quarter_end(d: date) -> date:
start = _quarter_start(d)
return _month_end(date(start.year, start.month + 2, 1))
def _as_date(value: Any) -> date | None:
"""日期写法的期次 → 那一天;带非零点时刻的、认不出的为 None。"""
if isinstance(value, datetime):
return value.date() if value.time() == datetime.min.time() else None
if isinstance(value, date):
return value
if isinstance(value, bool):
return None
day = coverage.as_day(value)
if day is None or day.timed:
return None
return day.day
def _shape(value: Any) -> str | None:
"""一个期次值是哪种写法:date / month / week / quarter / seq;认不出为 None。"""
if isinstance(value, bool) or value is None:
return None
if isinstance(value, (date, datetime)):
return "date" if _as_date(value) else None
if isinstance(value, float) and value.is_integer():
value = int(value)
if isinstance(value, int):
text = str(value)
if len(text) == 8 and _as_date(value):
return "date"
if len(text) == 6 and (m := _MONTH.match(text)) and 1 <= int(m.group(2)) <= 12:
return "month"
return "seq"
if not isinstance(value, str):
return None
text = value.strip()
if (m := _MONTH.match(text)) and 1 <= int(m.group(2)) <= 12 and 1900 <= int(m.group(1)) <= 2200:
return "month"
if _WEEK.match(text):
return "week"
if _QUARTER.match(text):
return "quarter"
if _as_date(text):
return "date"
if re.fullmatch(r"-?\d+", text):
return "seq"
return None
def _month_of(value: Any) -> date:
m = _MONTH.match(str(int(value) if isinstance(value, (int, float)) else value).strip())
assert m is not None
return date(int(m.group(1)), int(m.group(2)), 1)
def _infer_date_grain(days: Sequence[date]) -> str:
"""一列日期的粒度(见文件头)。"""
ordered = sorted(set(days))
if len(ordered) < 2:
return "day"
firsts = all(d.day == 1 for d in ordered)
lasts = all(d == _month_end(d) for d in ordered)
if firsts or lasts:
months = [d.year * 12 + d.month - 1 for d in ordered]
gaps = [b - a for a, b in zip(months, months[1:])]
if all(g >= 1 for g in gaps):
if all(d.month in (1, 4, 7, 10) for d in ordered) and all(g % 3 == 0 for g in gaps) and firsts:
return "quarter"
return "month"
gaps = [(b - a).days for a, b in zip(ordered, ordered[1:])]
if all(g % 7 == 0 for g in gaps):
return "week"
return "day"
def parse(values: Sequence[Any], grain: str | None = None) -> tuple[list[Period], str] | None:
"""一列期次值 → (逐个的 Period,粒度)。认不出(写法不一、带时刻、粒度和写法对不上)返回 None。"""
shapes = {_shape(v) for v in values}
if not values or None in shapes or len(shapes) != 1:
return None
shape = shapes.pop()
if shape == "seq":
if grain not in (None, "seq"):
return None
return [Period(v, int(v), None, None) for v in values], "seq"
if shape == "month":
if grain not in (None, "month"):
return None
out = []
for v in values:
start = _month_of(v)
out.append(Period(v, start.year * 12 + start.month - 1, start, _month_end(start)))
return out, "month"
if shape == "week":
if grain not in (None, "week"):
return None
out = []
for v in values:
m = _WEEK.match(str(v).strip())
assert m is not None
try:
start = date.fromisocalendar(int(m.group(1)), int(m.group(2)), 1)
except ValueError:
return None
out.append(Period(v, start.toordinal() // 7, start, start + timedelta(days=6)))
return out, "week"
if shape == "quarter":
if grain not in (None, "quarter"):
return None
out = []
for v in values:
m = _QUARTER.match(str(v).strip())
assert m is not None
start = date(int(m.group(1)), 3 * (int(m.group(2)) - 1) + 1, 1)
out.append(Period(v, start.year * 4 + (start.month - 1) // 3, start, _quarter_end(start)))
return out, "quarter"
days = [_as_date(v) for v in values]
assert all(days)
grain = grain or _infer_date_grain(days) # type: ignore[arg-type]
if grain == "day":
return [Period(v, d.toordinal(), d, d) for v, d in zip(values, days)], "day" # type: ignore[union-attr]
if grain == "week":
first = min(days) # type: ignore[type-var]
if any((d - first).days % 7 for d in days): # type: ignore[operator]
return None
return [Period(v, (d - first).days // 7, d, d + timedelta(days=6)) # type: ignore[operator]
for v, d in zip(values, days)], "week"
if grain == "month":
if not all(d.day == 1 or d == _month_end(d) for d in days): # type: ignore[union-attr]
return None
return [Period(v, d.year * 12 + d.month - 1, d.replace(day=1), _month_end(d)) # type: ignore[union-attr]
for v, d in zip(values, days)], "month"
if grain == "quarter":
if not all(d.day == 1 and d.month in (1, 4, 7, 10) for d in days): # type: ignore[union-attr]
return None
return [Period(v, d.year * 4 + (d.month - 1) // 3, d, _quarter_end(d)) # type: ignore[union-attr]
for v, d in zip(values, days)], "quarter"
return None
def calendar_unit(start: date, end: date) -> str | None:
"""这段日子是不是一个整的日历期:整月 month、整季 quarter、整年 year;都不是为 None。整月对整月算一样长(31 天的 10 月
和 30 天的 9 月环比是通行的比法),不按天数判窗口不等。"""
if start.day != 1:
return None
if start.month == 1 and end == date(start.year, 12, 31):
return "year"
if start.month in (1, 4, 7, 10) and end == _quarter_end(start):
return "quarter"
if end == _month_end(start):
return "month"
return None
def weekend_days(start: date, end: date) -> int:
"""[start, end] 里周六、周日的天数。"""
days = (end - start).days + 1
full, rest = divmod(days, 7)
count = 2 * full
for k in range(rest):
if (start + timedelta(days=7 * full + k)).weekday() >= 5:
count += 1
return count
__all__ = ["COUNT_WORD", "DEFAULT_LAG", "GRAINS", "PER_WORD", "Period", "calendar_unit", "parse", "weekend_days"]
+517
View File
@@ -0,0 +1,517 @@
"""判定语、判定标准、附带说明:一种说法,只在这里定义(细案 01 §7)。
判定语由系统渲染([[a:A3]] 渲染成结论里的 phrase),写作者只引用、不改写——「叙述层无算术」往前一步:分析判定和数字一样
由系统写。前后端都只读结论里渲染好的字符串,所以模板、数的写法、附带说明的那一句都在这里,算子只填关节。
**数的写法**(和 engine/evidence.render_cell 同一种风格:Decimal 式的四舍五入 ROUND_HALF_UP,千分位用「,」):
- 百分比:一位小数、带正负号(+8.7%、-3.1%,0 写 0.0%);占比、比重不带正号(占 61.9%)。不是 0 的值写出来成了 0.0%
的,多给小数位(最多 4 位),不写一个假的 0。
- 量(金额、件数):千分位,最多两位小数、去尾零;绝对值小于 1 的保留两位有效数字(最多 6 位小数),同样不写假的 0。
- 百分点:比率的变化(2.1% 到 2.5% 是上升 0.4 个百分点)。
**label 消毒**(§7.3):判定语里的度量叫法、因子名、维度叫法是 agent 或数据给的。agent 给的 label(以及目录里的叫法、
当叫法用的列名)去掉数字、百分号、单位符号、引用标记,不超过 20 字,消毒后为空就退回下一个来源:不然模型可以把
「销售额(同比 +30%)」塞进 label,借系统渲染绕过裸数字检测。去数字按出具校验抽数字的同一个函数(engine/issuance 的
extract_numbers:阿拉伯数字、中文数字、带万 / 亿 / 成 / 倍的写法),再去掉剩下的数字字符。维度值是数据,照原样。
**附带说明**:判定语后面用「;」接,最多两条(按 PHRASE_ORDER 的先后),其余只在证据面板。度量类型没标、以各段之和为
总数这类方法上的交代不进判定语。
"""
from __future__ import annotations
import math
import re
from collections.abc import Iterable, Mapping, Sequence
from fractions import Fraction
from typing import Any
from app.analysis.numbers import fixed, to_fraction
#: 算子的叫法:「算不了」的判定语、报错都用它
OP_NAMES = {"decompose": "变化拆解", "baseline": "波动基线", "trend": "趋势判断", "differ": "差异检验",
"comparable": "可比性检查"}
#: 附带说明进判定语的先后;不在这里的(measure_unlabeled、headline_basis)只在证据面板
PHRASE_ORDER = ("comparability", "simpson", "offset", "outlier_sensitive", "small_sample", "record_high", "record_low",
"multiple_comparisons", "weekday_effect", "small_denominator", "null_values")
#: 判定语里最多接几条附带说明
PHRASE_CAVEATS = 2
#: 消毒后的叫法最多几个字
LABEL_MAX = 20
#: 维度键有两列时,一段的写法(华东·线上)
KEY_JOIN = "·"
#: 维度值为空时的写法
EMPTY_KEY = "(空)"
# --------------------------------------------------------------------------
# 拼字:汉字和英文、数字之间加一个空格,挨着全角标点的不加(同 data/sql_semantics._say)
# --------------------------------------------------------------------------
_CJK_PUNCT = frozenset(",。;:、!?「」『』()《》【】…—·")
def _wide(ch: str) -> bool:
return "⺀" <= ch <= "鿿" or "豈" <= ch <= "﫿" or "＀" <= ch <= "￯"
def say(*parts: str | None) -> str:
"""拼一句话:汉字和英文、数字之间补一个空格,挨着全角标点、空白的不补,汉字和汉字之间不补。"""
out = ""
for part in parts:
if not part:
continue
if out and not out[-1].isspace() and not part[0].isspace():
a, b = out[-1], part[0]
if not (a in _CJK_PUNCT or b in _CJK_PUNCT or (_wide(a) and _wide(b)) or (not _wide(a) and not _wide(b))):
out += " "
out += part
return out
# --------------------------------------------------------------------------
# 数
# --------------------------------------------------------------------------
def _q(value: Any) -> Fraction:
q = value if isinstance(value, Fraction) else to_fraction(value)
if q is None:
raise ValueError(f"不是数:{value!r}")
return q
def _nonzero_fixed(q: Fraction, places: int, *, most: int, strip: bool, group: bool = False) -> str:
"""按 places 位小数写;不是 0 的值写出来成了 0 的,逐位多给,最多 most 位。"""
for p in range(places, most + 1):
text = fixed(q, p, strip=strip if p == places else True, group=group)
if q == 0 or text.strip("-0.,"):
return text
return fixed(q, most, strip=True, group=group)
def pct(value: Any, *, signed: bool = True) -> str:
"""比率写成百分比:一位小数;signed 时正数带「+」(0 不带)。"""
q = _q(value) * 100
text = _nonzero_fixed(q, 1, most=4, strip=False)
if signed and q > 0 and not text.startswith("-"):
text = "+" + text
return text + "%"
def points(value: Any) -> str:
"""比率的变化写成百分点(不带符号,调用方写上升 / 下降):0.004 → 0.4。"""
return _nonzero_fixed(abs(_q(value)) * 100, 1, most=4, strip=True)
def amount(value: Any, *, signed: bool = False) -> str:
"""量:千分位,最多两位小数、去尾零;绝对值小于 1 的保留两位有效数字(最多 6 位小数)。signed 时正数带「+」。"""
q = _q(value)
places = 2
if q != 0 and abs(q) < 1:
places = max(2, -math.floor(math.log10(float(abs(q)))) + 1)
places = min(places, 6)
text = _nonzero_fixed(q, places, most=6, strip=True, group=True)
if signed and q > 0:
text = "+" + text
return text
def threshold_pct(value: Any) -> str:
"""阈值写成百分比:去尾零、不带正号(0.005 → 0.5%,0.8 → 80%)。"""
return fixed(_q(value) * 100, 4, strip=True) + "%"
def share_words(value: Any) -> str:
"""阈值写进判定标准:0.5 写「一半」,其余写百分比。"""
q = _q(value)
return "一半" if q == Fraction(1, 2) else threshold_pct(q)
def number(value: Any) -> str:
"""阈值本身(倍数、显著性水平、期数):去尾零、不带千分位。"""
return fixed(_q(value), 6, strip=True)
def change(delta: Any, base: Any) -> str:
"""一个量的变化:下降 194(-8.7%)/ 上升 120(+5.3%);基期为 0 不给百分比;没变写「没有变化」。"""
d = _q(delta)
if d == 0:
return "没有变化"
word = "上升" if d > 0 else "下降"
b = _q(base) if base is not None else Fraction(0)
tail = f"({pct(d / abs(b))})" if b != 0 else ""
return f"{word} {amount(abs(d))}{tail}"
def ratio_change(r_base: Any, r_target: Any, *, proportion: bool) -> str:
"""一个比率的变化。proportion(各段的分子都不超过分母,比率是一个占比,如退款率)写百分点:上升 0.4 个百分点(2.1% 到
2.5%);其余(客单价这类)按量写:上升 3.2(+3.9%)。"""
rb, rt = _q(r_base), _q(r_target)
d = rt - rb
if not proportion:
return change(d, rb)
if d == 0:
return f"没有变化({pct(rt, signed=False)})"
word = "上升" if d > 0 else "下降"
return f"{word} {points(d)} 个百分点({pct(rb, signed=False)} 到 {pct(rt, signed=False)})"
def ratio_value(value: Any, *, proportion: bool) -> str:
"""比率本身:占比写百分比,其余按量写。"""
return pct(value, signed=False) if proportion else amount(value)
# --------------------------------------------------------------------------
# 叫法与段
# --------------------------------------------------------------------------
_MARKER = re.compile(r"\[\[[^\]\n]*\]\]|\[\[|\]\]")
_DIGITS = re.compile(r"[0-90-9]")
_SYMBOLS = re.compile(r"[%%‰‱¥¥$$€£₩℃°++\-−–±×÷==<><>~~^*]")
_EMPTY_BRACKETS = re.compile(r"[((【\[]\s*[))】\]]")
_EDGE_PUNCT = ",,、::;;。.!!??  "
def clean_label(text: Any) -> str:
"""消毒(见文件头):去引用标记、数字(出具校验认的数字写法连同后缀,再加剩下的数字字符)、百分号和单位符号、空括号,
压掉多余空白,不超过 LABEL_MAX 字。可能消成空串。"""
if not isinstance(text, str):
return ""
from app.engine.issuance import extract_numbers
raw = _MARKER.sub("", text)
spans = sorted((n.start, n.end) for n in extract_numbers(raw))
kept, pos = [], 0
for start, end in spans:
if start >= pos:
kept.append(raw[pos:start])
pos = end
kept.append(raw[pos:])
out = _SYMBOLS.sub("", _DIGITS.sub("", "".join(kept)))
# 去掉数字以后留下的空白、标点:括号、标点两边的空白,括号里头尾的标点,连着的几个标点
out = re.sub(r"\s*([()()【】\[\],,、::;;])\s*", r"\1", out)
out = re.sub(r"[,,、::;;]+(?=[))】\]])", "", out)
out = re.sub(r"(?<=[((【\[])[,,、::;;]+", "", out)
out = re.sub(r"([,,、::;;])[,,、::;;]+", r"\1", out)
while _EMPTY_BRACKETS.search(out):
out = _EMPTY_BRACKETS.sub("", out)
out = re.sub(r"\s+", " ", out).strip(_EDGE_PUNCT)
return out[:LABEL_MAX].strip(_EDGE_PUNCT)
def pick_label(*candidates: Any, fallback: str = "这一列") -> str:
"""依次取第一个消毒后不为空的叫法;都为空用 fallback。"""
for c in candidates:
cleaned = clean_label(c)
if cleaned:
return cleaned
return fallback
def key_text(key: Sequence[Any] | Any) -> str:
"""一段(维度键)的写法:一列就是那个值,两列用「·」连(华东·线上);空值写「(空)」。数据里的值照原样(压成一行)。"""
parts = key if isinstance(key, (list, tuple)) else [key]
shown = []
for p in parts:
if p is None:
shown.append(EMPTY_KEY)
elif isinstance(p, bool):
shown.append("是" if p else "否")
else:
shown.append(re.sub(r"\s+", " ", str(p)).strip() or EMPTY_KEY)
return KEY_JOIN.join(shown)
def names(items: Iterable[str], limit: int | None = None) -> str:
"""几段的写法:华东、华北;超过 limit 段时写前 limit 段加「等 n 段」。"""
items = list(items)
if limit is not None and len(items) > limit:
return "、".join(items[:limit]) + f"等 {len(items)} 段"
return "、".join(items)
def dims_text(labels: Sequence[str]) -> str:
"""维度的叫法:地区;两列时写「地区、渠道」。"""
return "、".join(labels)
# --------------------------------------------------------------------------
# 附带说明(caveats[].text):一种说法
# --------------------------------------------------------------------------
def caveat(code: str, text: str) -> dict[str, str]:
return {"code": code, "text": text}
def offset_text(delta_total: Any, gross_up: Any, gross_down: Any) -> str:
"""抵消:和总变化方向相反的段合计多少(下降的段合计 -120,上升的段合计 +40,互相抵消了一部分)。"""
up, down = amount(gross_up, signed=True), amount(gross_down, signed=True)
if _q(delta_total) >= 0:
return f"上升的段合计 {up},下降的段合计 {down},互相抵消了一部分"
return f"下降的段合计 {down},上升的段合计 {up},互相抵消了一部分"
def simpson_text(label: str, overall_up: bool, subject: str, w_base: Any, w_target: Any) -> str:
"""辛普森:各段的比率大多和整体反着变,整体的变化来自结构。"""
seg = "上升" if not overall_up else "下降"
whole = "上升" if overall_up else "下降"
move = "升" if _q(w_target) > _q(w_base) else "降"
return say(f"各段的", label, f"大多在{seg},整体却{whole}:变化来自结构,", subject,
f"占比从 {pct(w_base, signed=False)} {move}到 {pct(w_target, signed=False)}")
def small_denominator_text(segments: Sequence[str], denominator: str, minimum: int) -> str:
return say(names(segments, 3), "的", denominator, f"不到 {minimum},比率波动大,不参与多数判断")
def null_segments_text(segments: Sequence[str], label: str) -> str:
return say(names(segments, 3), "的", label, "为空,按 0 算")
def null_periods_text(count: int, label: str) -> str:
return say(f"有 {count} 期的", label, "为空,跳过了这几期")
def unlabeled_text(column: str, tail: str) -> str:
return say("目录没标「", column, "」的度量类型,", tail)
def unconfirmed_text(column: str, measure: str, tail: str) -> str:
return say("目录里「", column, f"」是{MEASURE_WORDS.get(measure, measure)}只是推断,没人确认,", tail)
MEASURE_WORDS = {"flow": "流量", "stock": "存量", "ratio": "比率", "identifier": "标识", "status": "状态", "attribute": "属性"}
HEADLINE_BASIS = "没给总数,以各段之和为总数"
WEEKDAY_EFFECT = "日数据有星期效应,看趋势建议按周汇总"
def record_text(high: bool, label: str, n: int, unit: str) -> str:
return say("本期的", label, f"是过去 {n} {unit}里{'最高' if high else '最低'}的")
def multiple_text(pairs: int) -> str:
return f"比了 {pairs} 对,已按次数校正"
def outlier_text(largest: bool, now_differs: bool) -> str:
which = "最大" if largest else "最小"
return f"去掉{which}的一笔后,差异{'变得站得住' if now_differs else '不再站得住'}"
def small_sample_rate_text(minimum: int) -> str:
return f"样本少(有格子的期望频数不到 {minimum}),用了精确检验"
def small_sample_mean_text(group: str, n: int, minimum: int) -> str:
return say("有一组的样本数不到 ", str(minimum), "(", group, f" {n}),结论不稳")
UNVERIFIED_COMPLETE = "没能核对本期的数据是否到齐(查不到表里这一列的日期范围)"
def with_caveats(main: str, caveats: Sequence[Mapping[str, str]], *, skip: Iterable[str] = ()) -> str:
"""判定语接上最多 PHRASE_CAVEATS 条附带说明(按 PHRASE_ORDER 的先后,用「;」接)。skip 里的那几条(已经在正文里说过的、
只在证据面板说的)不接。"""
skipped = set(skip)
ranked = sorted((PHRASE_ORDER.index(c["code"]), i, c["text"]) for i, c in enumerate(caveats)
if c["code"] in PHRASE_ORDER and c["code"] not in skipped)
texts = list(dict.fromkeys(t for _, _, t in ranked))[:PHRASE_CAVEATS]
return ";".join([main, *texts])
def refused(op: str, reason: str) -> str:
"""算不了的判定语:{算子的叫法}没有做成:{原因}。"""
return f"{OP_NAMES[op]}没有做成:{reason}"
# --------------------------------------------------------------------------
# 判定语模板(§7.2)
# --------------------------------------------------------------------------
def decompose_driver(label: str, total: str, subject: str, subject_change: str, share: Any) -> str:
return say(label, total, ",主要来自", subject, f"({subject_change},占 {pct(share, signed=False)})")
def decompose_concentrated(label: str, total: str, subjects: str, share: Any) -> str:
return say(label, total, ",集中在", subjects, f"(合计占 {pct(share, signed=False)})")
def decompose_dispersed(label: str, total: str, dims: str, main_share: Any) -> str:
return say(label, total, ",分散在各", dims, ",没有哪一段占到", share_words(main_share))
def decompose_flat(label: str, pct_total: Any | None, offset: tuple[str, str, str, str] | None) -> str:
"""基本持平;有抵消时接「,内部此消彼长:华东 +120、华北 -118」(最大的上升段、最大的下降段)。"""
head = say(label, "基本持平") + (f"({pct(pct_total)})" if pct_total is not None else "")
if offset is None:
return head
up_key, up_change, down_key, down_change = offset
return head + say(",内部此消彼长:", up_key, f" {up_change}、", down_key, f" {down_change}")
def decompose_unclosed(dims: str, gap: Any, *, ratio: bool) -> str:
what = "各段合起来的比率和总数对不上" if ratio else "各段之和和总数对不上"
return say("按", dims, f"拆开后{what}(差 {amount(abs(_q(gap)))}),不判断变化来自哪里")
def ratio_mix(label: str, total: str, subject: str, w_base: Any, w_target: Any) -> str:
return say(label, total, ",主要来自结构变化:", subject,
f"占比从 {pct(w_base, signed=False)} 到 {pct(w_target, signed=False)}")
def ratio_rate(label: str, total: str, subject: str) -> str:
return say(label, total, ",主要来自各段自身的变化,", subject, "影响最大")
def ratio_both(label: str, total: str, mix_share: Any, rate_share: Any) -> str:
return say(label, total, f",结构变化和各段自身变化都有影响(各占 {pct(mix_share, signed=False)}、"
f"{pct(rate_share, signed=False)})")
def factor_driver(label: str, total: str, factor: str, contrib: Any, share: Any) -> str:
return say(label, total, ",主要来自", factor, f"的变化(贡献 {amount(contrib, signed=True)},占 "
f"{pct(share, signed=False)})")
def factor_both(label: str, total: str, parts: Sequence[tuple[str, Any]]) -> str:
"""乘积拆解里没有哪个因子占到「主要」:列出各因子占的比重。"""
shown = "、".join(say(name, f"占 {pct(share, signed=False)}") for name, share in parts)
return say(label, total, f",几个因子都有影响({shown})")
def band_text(lo: Any, hi: Any, *, basis: str) -> str:
fmt = (lambda v: pct(v)) if basis == "change" else (lambda v: amount(v))
return f"{fmt(lo)} 至 {fmt(hi)}"
def current_text(x: Any, *, basis: str, lag: int, grain: str) -> str:
"""本期:变化率(和比较期比)或值本身。日数据 lag 7 写「比上周同一天」,lag 52 周 / 12 月 / 4 季写「同比」。"""
if basis == "level":
return f"本期为 {amount(x)}"
if lag == 1:
lead = "本期变化"
elif grain == "day" and lag == 7:
lead = "比上周同一天变化"
elif (grain, lag) in (("week", 52), ("month", 12), ("quarter", 4)):
lead = "同比变化"
else:
lead = f"比 {lag} 期前变化"
return f"{lead} {pct(x)}"
def baseline_phrase(verdict: str, label: str, current: str, n: int, unit: str, band: str) -> str:
where = {"within_band": "在过去 {n} {u}的常规波动范围内", "outside_band": "超出过去 {n} {u}的常规波动范围",
"far_outside_band": "明显超出过去 {n} {u}的常规波动范围"}[verdict].format(n=n, u=unit)
return say(label, current, f",{where}({band})")
def baseline_flat_phrase(verdict: str, label: str, current: str, n: int, unit: str, per: str, value: str) -> str:
"""历史每期都一样(稳健标准差为 0)。"""
if verdict == "within_band":
return say(label, current, f",和过去 {n} {unit}一样(每{per}都是 {value})")
return say(label, current, f",明显超出过去 {n} {unit}的常规波动范围(每{per}都是 {value})")
def trend_phrase(verdict: str, label: str, n: int, unit: str, per: str, slope: str | None,
run: tuple[int, str] | None) -> str:
if verdict == "no_trend":
return say(f"过去 {n} {unit}", label, "没有明显的上升或下降趋势")
word = "上升" if verdict == "rising" else "下降"
text = say(f"过去 {n} {unit}", label, f"呈{word}趋势") + (f"(每{per}约 {slope})" if slope else "")
if run is not None:
length, direction = run
text += f",已连续 {length} {unit}{'上升' if direction == 'up' else '下降'}"
return text
def differ_pair(a: str, b: str, label: str, va: str, vb: str, higher: bool) -> str:
"""一对的差异站得住:西南的退款率(5.0%)高于华东(2.1%)。"""
return say(a, "的", label, f"({va}){'高于' if higher else '低于'}", b, f"({vb})")
def differ_phrase(pairs: Sequence[str], total: int) -> str:
"""差异站得住。比了几对时写「比了 4 对,2 对差异站得住:…」。"""
if total == 1:
return f"{pairs[0]},差异站得住"
shown = ";".join(pairs[:2]) + ("等" if len(pairs) > 2 else "")
return f"比了 {total} 对,{len(pairs)} 对差异站得住:{shown}"
def no_evidence_phrase(a: str | None, b: str | None, label: str, sizes: tuple[str, str] | None, total: int) -> str:
if total == 1 and a is not None and b is not None and sizes is not None:
return say(a, "和", b, "的", label, f"差异可能只是样本波动(样本量 {sizes[0]}、{sizes[1]})")
return say(f"比了 {total} 对,", label, "的差异可能都只是样本波动")
def comparable_phrase(verdict: str, reasons: Sequence[str], unknown: Sequence[str]) -> str:
if verdict == "incomparable":
return "两期不可比:" + ";".join(reasons)
if verdict == "conditional":
return "两期有条件可比:" + ";".join(reasons)
return "两期可比" + (f",{'、'.join(unknown)}核对不了" if unknown else "")
# --------------------------------------------------------------------------
# 判定标准(criterion):照实际阈值写的一句
# --------------------------------------------------------------------------
def criterion_additive(s: Mapping[str, Any], *, top_k_max: int | None, summed: bool) -> str:
k = f"最多前 {top_k_max} 段" if top_k_max else "前几段"
text = say("一段占总变化的", share_words(s["main_share"]), "以上算主要来源,", k, "合计占",
share_words(s["concentration"]), "以上算集中,总变化不到基期的", threshold_pct(s["flat_pct"]),
"算基本持平")
return text + (",以各段之和为总数" if summed else ",各段之和要和总数对得上才判断变化来自哪里")
def criterion_ratio(s: Mapping[str, Any]) -> str:
return say("结构变化或各段自身变化占比率变化的", share_words(s["main_share"]), "以上算主要来自它,比率变化不到基期的",
threshold_pct(s["flat_pct"]), f"算基本持平,分母不到 {number(s['min_denominator'])} 的段不参与多数判断")
def criterion_factor(s: Mapping[str, Any]) -> str:
return say("按对数平均拆分(LMDI),一个因子的贡献占总变化的", share_words(s["main_share"]),
"以上算主要来自它,总变化不到基期的", threshold_pct(s["flat_pct"]), "算基本持平")
def criterion_baseline(s: Mapping[str, Any], n: int | None, unit: str) -> str:
span = f"过去 {n} {unit}" if n else f"过去最多 {number(s['max_history'])} 期"
warn, alert = number(s["k_warn"]), number(s["k_alert"])
return (f"本期和{span}的中位数比,差不到 {warn} 倍稳健标准差(1.4826 倍的中位绝对偏差)算常规波动,{warn} 到 "
f"{alert} 倍算超出,{alert} 倍以上算明显超出;历史至少要 {number(s['min_history'])} 期")
def criterion_trend(s: Mapping[str, Any], n: int | None, unit: str) -> str:
span = f"最后 {n} {unit}" if n else f"最后最多 {number(s['max_history'])} 期"
return (f"用{span}(至少 {number(s['trend_min_points'])} 期):Mann–Kendall 检验的双侧 p 值小于 "
f"{number(s['trend_alpha'])} 时按 Theil–Sen 斜率的正负判上升或下降,否则判没有明显趋势")
def criterion_differ(s: Mapping[str, Any], mode: str) -> str:
alpha = number(s["differ_alpha"])
if mode == "rate":
return (f"双侧 p 值(比了几对时按 Holm 校正)小于 {alpha} 算差异站得住;有格子的期望频数不到 "
f"{number(s['min_expected'])} 时用 Fisher 精确检验,否则用合并比例的 z 检验")
return (f"Welch t 检验的双侧 p 值(比了几对时按 Holm 校正)小于 {alpha} 算差异站得住;一组样本数不到 "
f"{number(s['min_n'])} 算样本少")
CRITERION_COMPARABLE = ("只差日期、窗口一样长、是整期、星期构成相同、两边查询都没有数据问题才算可比;只有提醒项算有条件"
"可比,有一项不通过算不可比;核对不了的项不判不可比")
__all__ = [
"CRITERION_COMPARABLE", "EMPTY_KEY", "HEADLINE_BASIS", "KEY_JOIN", "LABEL_MAX", "MEASURE_WORDS", "OP_NAMES",
"PHRASE_CAVEATS", "PHRASE_ORDER", "UNVERIFIED_COMPLETE", "WEEKDAY_EFFECT", "amount", "band_text",
"baseline_flat_phrase", "baseline_phrase", "caveat", "change", "clean_label", "comparable_phrase",
"criterion_additive", "criterion_baseline", "criterion_differ", "criterion_factor", "criterion_ratio",
"criterion_trend", "current_text", "decompose_concentrated", "decompose_dispersed", "decompose_driver",
"decompose_flat", "decompose_unclosed", "differ_pair", "differ_phrase", "dims_text", "factor_both",
"factor_driver", "key_text", "multiple_text", "names", "no_evidence_phrase", "null_periods_text",
"null_segments_text", "number", "offset_text", "outlier_text", "pct", "pick_label", "points", "ratio_both",
"ratio_change", "ratio_mix", "ratio_rate", "ratio_value", "record_text", "refused", "say", "share_words",
"simpson_text", "small_denominator_text", "threshold_pct", "small_sample_mean_text", "small_sample_rate_text", "trend_phrase",
"unconfirmed_text", "unlabeled_text", "with_caveats",
]
+254
View File
@@ -0,0 +1,254 @@
"""可比性检查的现查(细案 01 §1「只读快照、不发查询」的唯一例外,§6):表里这一列的实际日期范围、按运行当天现算的界。
算子本身不查库。要现查的只有两样,都走 data/coverage 现成的只读探查(只读守卫、遮罩列不探、每条 PROBE_TIMEOUT_S 秒、总
时限、同一张表 5 分钟内不重查、现查不成退回剖析事实):
- 范围:(数据源名, 表, 列) → contract.Probe {source, table, column, min, max, basis, at},min / max 是 ISO 日期;表里这一列
一个值都没有时 min、max 都为 None(basis live)。basis 是 live(现查)/ profile(按剖析)/ observed(按取值观察),at 是现查的
时刻(服务端时区,ISO,秒)或剖析、观察的日期。
- 界:SQL 里按运行当天现算的界(CURRENT_DATE、DATE_SUB(…)、date('now', …))在同一个库里算出来:(数据源名, 界的 SQL 写法)
→ Probe {source, expr, value, basis: live, at},value 是库里算出的日期(带时刻的写成「YYYY-MM-DD HH:MM:SS」)。
**结论里记下用到的现查结果,复算用记下的值**:Book 是一次分析的现查簿。run 时算子第一遍只登记要查什么(拿到的都是「不
知道」),入口查完再算一遍,直到不再要新的(算子在拿到范围之后才可能要界,最多三轮);复算时 Book 里装的是结论记下的
probes,不现查。结论的 probes 只记最后一遍真用到、查得到的,按用到的先后——同样的输入、同样的现查结果,结论逐字节相同。
入口(agent 分析工具、分析卡、口径卡的可比性挂钩)用 make_prober 生成 contract.Prober 交进 context["probe"],见它的说明。
"""
from __future__ import annotations
import time
from collections.abc import Awaitable, Callable, Iterable, Mapping
from datetime import datetime
from typing import Any
from app.analysis.contract import Probe
from app.data.names import name_key
#: 算子在拿到现查结果之后可能还要新的(先要范围、再要界):run 最多这么多轮
MAX_ROUNDS = 3
def _range_key(source: str, table: str, column: str) -> tuple[str, str, str, str]:
return ("range", source or "", name_key(str(table)), name_key(str(column)))
def _bound_key(source: str, expr: str) -> tuple[str, str, str]:
return ("bound", source or "", str(expr))
def probe_key(probe: Mapping[str, Any]) -> tuple[str, ...]:
"""一条记下的现查结果的键(Book 按它查)。"""
if probe.get("expr") is not None:
return _bound_key(str(probe.get("source") or ""), str(probe["expr"]))
return _range_key(str(probe.get("source") or ""), str(probe.get("table") or ""), str(probe.get("column") or ""))
class Book:
"""一次分析的现查簿。known:{键: Probe 或 None(查过、查不到)}。算子经 range / bound 取;asked 是按先后登记的键,missing
是登记了、簿里还没有的(run 据此去查),used 是最后真用到、查得到的(写进结论的 probes)。"""
def __init__(self, known: Mapping[tuple[str, ...], Probe | None] | Iterable[Probe] = ()) -> None:
if isinstance(known, Mapping):
self._known: dict[tuple[str, ...], Probe | None] = dict(known)
self._closed = False
else:
self._known = {probe_key(p): p for p in known}
self._closed = True # 复算:记下的就是全部,没记的当查不到
self.asked: list[tuple[str, ...]] = []
def _ask(self, key: tuple[str, ...]) -> Probe | None:
if key not in self.asked:
self.asked.append(key)
return self._known.get(key)
def range(self, source: str, table: str, column: str) -> Probe | None:
return self._ask(_range_key(source, table, column))
def bound(self, source: str, expr: str) -> Probe | None:
return self._ask(_bound_key(source, expr))
@property
def missing(self) -> list[tuple[str, ...]]:
return [] if self._closed else [k for k in self.asked if k not in self._known]
@property
def used(self) -> list[Probe]:
return [p for p in (self._known.get(k) for k in self.asked) if p is not None]
def _now() -> str:
return datetime.now().astimezone().replace(tzinfo=None).isoformat(timespec="seconds")
async def fetch(book: Book, context: Mapping[str, Any]) -> dict[tuple[str, ...], Probe | None]:
"""把簿里登记了、还没有的现查一遍:先 context["probe"],查不到的范围退回 context["facts"](数据源名 → {表名: 剖析事实},
或数据源名 → 取剖析事实的函数),都没有记 None。交回新查到的那几条。"""
from app.data import coverage
prober = context.get("probe")
facts = context.get("facts") or {}
out: dict[tuple[str, ...], Probe | None] = {}
for key in book.missing:
found: Probe | None = None
if key[0] == "range":
_, source, table, column = key
if prober is not None:
try:
found = await prober(source, table, column)
except Exception: # noqa: BLE001 - 探查失败就当不知道
found = None
if found is None:
found = await _from_facts(facts.get(source), source, table, column, coverage)
else:
_, source, expr = key
bounds = getattr(prober, "bounds", None)
if bounds is not None:
try:
found = await bounds(source, expr)
except Exception: # noqa: BLE001
found = None
out[key] = found
return out
async def _from_facts(facts: Any, source: str, table: str, column: str, coverage: Any) -> Probe | None:
"""探查不成时按剖析事实兜底:facts 是 {表名: last_facts 的那一项} 或取它的函数(coverage.Facts)。"""
found: Any = None
try:
if callable(facts):
found = await facts(table)
elif isinstance(facts, Mapping):
found = next((v for k, v in facts.items() if name_key(str(k)) == name_key(table)
or name_key(str(k).rsplit(".", 1)[-1]) == name_key(table.rsplit(".", 1)[-1])), None)
except Exception: # noqa: BLE001
return None
data = coverage.facts_range(found, column)
return _range_probe(source, table, column, data) if data else None
def _range_probe(source: str, table: str, column: str, data: Mapping[str, Any]) -> Probe:
basis = data.get("basis") if data.get("basis") in ("live", "profile", "observed") else "live"
at = _now() if basis == "live" else str(data.get("at") or "")
return {"source": source, "table": table, "column": column, "min": data.get("min"), "max": data.get("max"),
"basis": basis, "at": at} # type: ignore[typeddict-item]
class LiveProber:
"""contract.Prober:await prober(数据源名, 表, 列) → Probe | None;另有 await prober.bounds(数据源名, 界的 SQL) 在库里算界,
prober.dialect(数据源名) 是这个源的 sqlglot 方言(算子按它读 SQL、写界)。见 make_prober。"""
def __init__(self, sources: Mapping[str, Any], *, masked: Mapping[str, Iterable[str]] | None,
facts: Mapping[str, Callable[[str], Awaitable[Mapping[str, Any] | None]]] | None,
cache: Mapping[str, Any] | None, budget: float,
clock: Callable[[], str] | None) -> None:
from app.data import coverage
self._sources = dict(sources)
self._masked = {k: {str(c).strip().lower() for c in v} for k, v in (masked or {}).items()}
self._facts = dict(facts or {})
self._cache: dict[str, Any] = dict(cache or {})
self._budget = budget
self._deadline: float | None = None
self._clock = clock or _now
self._bounds: dict[tuple[str, str], Any] = {}
self._coverage = coverage
def dialect(self, source: str) -> str | None:
from app.data.sqlcheck import dialect_of
found = self._sources.get(source)
return dialect_of(getattr(found, "kind", None)) if found is not None else None
def _expired(self) -> bool:
if self._deadline is None:
self._deadline = time.monotonic() + self._budget
return time.monotonic() > self._deadline
def _hidden(self, source: Any, name: str) -> set[str]:
if name in self._masked:
return self._masked[name]
from app.data.engine import masked_columns
return {c.lower() for c in masked_columns(getattr(source, "options", None) or {})}
def _facts_of(self, name: str, source: Any) -> Callable[[str], Awaitable[Mapping[str, Any] | None]]:
return self._facts.get(name) or profile_facts(source)
async def __call__(self, source: str, table: str, column: str) -> Probe | None:
found = self._sources.get(source)
if found is None or str(column).strip().lower() in self._hidden(found, source):
return None
if self._expired():
return None
cache = self._cache.setdefault(source, self._coverage.Cache())
data = await self._coverage.column_range(found, table, column, facts=self._facts_of(source, found), cache=cache)
if data is None:
return None
probe = _range_probe(source, table, column, data)
if probe["basis"] == "live":
probe["at"] = self._clock()
return probe
async def bounds(self, source: str, expr: str) -> Probe | None:
found = self._sources.get(source)
if found is None or self._expired():
return None
key = (source, expr)
if key not in self._bounds:
values = await self._coverage.bound_values(found, [expr])
self._bounds[key] = values[0] if values else None
value = self._bounds[key]
if value is None:
return None
return {"source": source, "expr": expr, "value": str(value), "basis": "live", # type: ignore[typeddict-unknown-key]
"at": self._clock()}
def profile_facts(source: Any) -> Callable[[str], Awaitable[Mapping[str, Any] | None]]:
"""这个数据源上一张表的剖析事实(catalog_profile_jobs.last_facts 的那一项,同 tools/datasource._coverage 的退回):上传
表格的快照对不上这次读的那一版时不用。"""
async def facts(table: str) -> Mapping[str, Any] | None:
from app.data import catalog as data_catalog
from app.data.catalog_profile_jobs import last_facts
from app.db.base import SessionLocal
source_id = getattr(source, "id", None)
if not source_id:
return None
name = data_catalog.resolve_table_name(getattr(source, "schema_cache", None), table) or table
async with SessionLocal() as session:
found = (await last_facts(session, str(source_id), [name])).get(name)
snapshot = getattr(source, "snapshot_id", None)
if found and snapshot and found.get("snapshot_id") != snapshot:
return None
return found
return facts
def make_prober(sources: Mapping[str, Any], *, masked: Mapping[str, Iterable[str]] | None = None,
facts: Mapping[str, Callable[[str], Awaitable[Mapping[str, Any] | None]]] | None = None,
cache: Mapping[str, Any] | None = None, budget: float | None = None,
clock: Callable[[], str] | None = None) -> LiveProber:
"""生成一次分析调用用的探查器(contract.Prober,交进 context["probe"])。
- sources:数据源名 → 数据源对象(DataSource 行,或上传源绑定快照的视图:同 tools/datasource 建工具时的 source,要有
name、kind、options、schema_cache;上传源另有 snapshot_id)。名字和快照里的 source 一致。没给的源不探。
- masked:数据源名 → 遮罩列(不分大小写)。缺省按数据源 options 里设的遮罩列(data/engine.masked_columns)。遮罩列上
不探,探查器直接交回 None。
- facts:数据源名 → 取剖析事实的函数(coverage.Facts:表名 → last_facts 的那一项)。缺省按 profile_facts 现读库(同查询
工具的覆盖检查)。
- cache:数据源名 → coverage.Cache(同一张表、同一个界一次运行里只查一次);不给就每个探查器各自一份。另外 coverage
在进程里把现查到的范围记 5 分钟。
- budget:这个探查器所有探查加起来最多多少秒(缺省 coverage.BUDGET_S,从第一次探查算起);过了的一律交回 None。一次
分析调用建一个。
- clock:现查时刻的写法(测试用);缺省是服务端时区的 ISO 时间(到秒)。
"""
from app.data import coverage
return LiveProber(sources, masked=masked, facts=facts, cache=cache,
budget=coverage.BUDGET_S if budget is None else budget, clock=clock)
__all__ = ["Book", "LiveProber", "MAX_ROUNDS", "fetch", "make_prober", "probe_key", "profile_facts"]
+300
View File
@@ -0,0 +1,300 @@
"""放回历史看(细案 01 §3、§4):波动基线(这个变化大不大、算不算异常)和趋势(是持续在变,还是一次波动)。
**参数**:query(一份快照:一条序列;要看某一段就在 SQL 里筛好)、period(期次列)、measure(度量列);可选 grain(day /
week / month / quarter,缺省按期次值推断,见 analysis/periods)、label。波动基线另有 target(本期,缺省最后一期)、basis
(change 缺省:和比较期比的变化率;level:值本身,给存量、比率用)、lag(和往前第几期比,缺省日数据 7、其余 1;同比写 52 /
12 / 4,历史要相应够长)。
**前提**(不满足就算不了,原因照实写):期次认得出(period_unreadable)、本期完整(同可比性检查的整期:没过完、数据没到、
SQL 截掉一部分 → partial_target;表的日期范围查不到时只按运行当天判,附一句核对不了)、本期和比较期之间不缺期(target_gap)、
比较期不为 0(zero_base)、历史够长(history_short);趋势要最后几期连着(中间缺期 series_gap)。度量格为空的那一期跳过。
**波动基线**:历史的中位数 m、中位绝对偏差 MAD,σ̂ = 1.4826·MAD,稳健 z = (x − m)/σ̂;常规波动范围 m ± k_warn·σ̂;全程
精确的有理数。σ̂ = 0(历史每期都一样)时本期一样判常规、不一样判明显超出。本期的值(不论 basis)是历史各期里最高 / 最低的
加 record_high / record_low:「历史新高」说的是值本身。为什么不用均值和标准差:促销周这类极端值会把它们拉宽。
**趋势**:最后 n 期(期次连着的那一段,trend_min_points ≤ n ≤ max_history + 1)的 Theil–Sen 斜率、Mann–Kendall 双侧 p,
相对斜率 = 斜率 / |序列中位数|。p < trend_alpha 时按斜率的正负判上升、下降。末尾连续同向的期数从最后一期往前数(变化为 0
断开)。已知局限:Mann–Kendall 假设各期独立,业务序列常有自相关,p 会偏小;所以缺省 n 至少 6、α 取 0.05,判定语只说
「呈上升趋势」,不说「显著上升」。日数据附一句星期效应。
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from fractions import Fraction
from typing import Any
from app.analysis import comparable as cmp
from app.analysis import periods as pd
from app.analysis import phrases as ph
from app.analysis import stats
from app.analysis.common import Env, build, column_label, label_for, need, refuse, sign_word, today_param
from app.analysis.contract import CallError
from app.analysis.inputs import Snap, same_value
from app.analysis.numbers import canon, to_fraction
#: 稳健标准差 = 1.4826 × MAD(正态时等于标准差)
MAD_SCALE = Fraction(14826, 10000)
#: lag 的上限(周数据同比是 52)
_LAG_MAX = 260
@dataclass
class Series:
snap: Snap
period: str
measure: str
grain: str
label: str
points: list[tuple[pd.Period, Fraction]] # 按期次排好、度量不为空的
nulls: int
params: dict[str, Any]
def _load(raw: Mapping[str, Any], env: Env, used: list[str]) -> Series | Any:
"""读出一条序列;算不了时直接交回 refused 的结论。"""
snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
period = snap.column(need(raw, "period", "期次列"), "期次列")
measure = snap.column(need(raw, "measure", "度量列"), "度量列")
grain = raw.get("grain")
if grain is not None and grain not in (*pd.GRAINS, "seq"):
raise CallError("bad_grain", "粒度只能写 day、week、month、quarter,或者不写(按期次值推断)")
params: dict[str, Any] = {"query": snap.ref, "period": period, "measure": measure}
label = label_for(env, raw.get("label"), snap, measure)
if raw.get("label") is not None:
params["label"] = ph.clean_label(raw.get("label")) or None
criterion = _criterion(env, None, "期")
env.inputs.use(snap, "data", [period, measure])
if snap.truncated:
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行,序列不完整",
criterion=criterion)
i_period = snap.index(period)
values = [row[i_period] if i_period < len(row) else None for row in snap.rows]
parsed = pd.parse(values, grain)
if parsed is None:
sample = "、".join(dict.fromkeys(ph.key_text([v]) for v in values[:3]))
return refuse(env, params, used, "period_unreadable",
f"期次列「{period}」的值认不出是哪一期({sample}):要日期、YYYY-MM、YYYY-Www、YYYY-Qn 或整数序号",
criterion=criterion)
items, grain = parsed
params["grain"] = grain
seen: set[int] = set()
points: list[tuple[pd.Period, Fraction]] = []
nulls = 0
for r, item in enumerate(items):
if item.ordinal in seen:
raise CallError("duplicate_rows", ph.say(f"{snap.ref} 里同一期有不止一行:在 SQL 里按 ",
column_label(env, snap, period), " 汇总,每期一行"))
seen.add(item.ordinal)
value = snap.number(r, measure, "度量列")
if value is None:
nulls += 1
continue
points.append((item, value))
points.sort(key=lambda p: p[0].ordinal)
return Series(snap, period, measure, grain, label, points, nulls, params)
def _criterion(env: Env, n: int | None, unit: str) -> str:
if env.op == "baseline":
return ph.criterion_baseline(env.settings, n, unit)
return ph.criterion_trend(env.settings, n, unit)
def _completeness(env: Env, series: Series, item: pd.Period) -> tuple[str, str | None]:
"""本期齐不齐(同可比性检查的整期):(pass / short / unknown, 不齐的那一句)。"""
series.params["today"] = today_param(env)
if item.start is None:
return "unknown", None
span = cmp.period_span("本期", series.snap, series.period, item, series.grain, series.measure, env.facts, env.book)
return cmp.span_coverage(span, env.today, env.book) # type: ignore[arg-type]
# --------------------------------------------------------------------------
# 波动基线
# --------------------------------------------------------------------------
def baseline(raw: Mapping[str, Any], env: Env) -> Any:
used = ["k_warn", "k_alert", "min_history", "max_history"]
loaded = _load(raw, env, used)
if not isinstance(loaded, Series):
return loaded
series, params = loaded, loaded.params
basis = raw.get("basis") or "change"
if basis not in ("change", "level"):
raise CallError("bad_basis", "比法只能写 change(和比较期比的变化率)或 level(值本身)")
lag = raw.get("lag", pd.DEFAULT_LAG[series.grain])
if isinstance(lag, bool) or not isinstance(lag, int) or not 1 <= lag <= _LAG_MAX:
raise CallError("bad_lag", f"往前比几期要写 1 到 {_LAG_MAX} 的整数(同比:周数据 52、月数据 12、季度 4)")
params.update(basis=basis, lag=lag)
unit, per = pd.COUNT_WORD[series.grain], pd.PER_WORD[series.grain]
criterion = _criterion(env, None, unit)
if not series.points:
return refuse(env, params, used, "history_short", f"只有 0 期历史,至少要 {int(env.settings['min_history'])} 期",
criterion=criterion)
if raw.get("target") is not None:
found = next((p for p in series.points if same_value(raw["target"], p[0].raw)), None)
if found is None:
raise CallError("period_not_found", f"期次列「{series.period}」里没有「{raw['target']}」(或那一期为空):"
"本期要写序列里有值的一期,不写就是最后一期")
else:
found = series.points[-1]
target, x_level = found
params["target"] = target.raw
caveats: list[dict[str, str]] = []
status, why = _completeness(env, series, target)
if status == "short":
return refuse(env, params, used, "partial_target", f"{why},和完整的{per}比没有意义", criterion=criterion)
if status == "unknown" and target.start is not None:
caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE))
by_ordinal = {p.ordinal: v for p, v in series.points}
before = [(p, v) for p, v in series.points if p.ordinal < target.ordinal]
if basis == "change":
prev = by_ordinal.get(target.ordinal - lag)
if prev is None:
return refuse(env, params, used, "target_gap",
f"本期和它的比较期(往前第 {lag} {'期' if series.grain == 'seq' else unit})之间缺期,没有比较期的值",
criterion=criterion)
if prev == 0:
return refuse(env, params, used, "zero_base", ph.say("比较期的", series.label, "为 0,算不了变化率"),
criterion=criterion)
x = (x_level - prev) / abs(prev)
history = [(p, (v - by_ordinal[p.ordinal - lag]) / abs(by_ordinal[p.ordinal - lag]), v) for p, v in before
if by_ordinal.get(p.ordinal - lag) not in (None, 0)]
else:
x = x_level
history = [(p, v, v) for p, v in before]
history = history[-int(env.settings["max_history"]):]
n = len(history)
criterion = _criterion(env, n, unit)
if n < int(env.settings["min_history"]):
return refuse(env, params, used, "history_short", f"只有 {n} 期历史,至少要 {int(env.settings['min_history'])} 期",
criterion=criterion)
hv = [h for _, h, _ in history]
m = stats.median(hv)
sigma = MAD_SCALE * stats.mad(hv, m)
k_warn, k_alert = env.threshold("k_warn"), env.threshold("k_alert")
if sigma == 0:
z = None
code = "within_band" if x == m else "far_outside_band"
else:
z = (x - m) / sigma
code = "within_band" if abs(z) < k_warn else "outside_band" if abs(z) < k_alert else "far_outside_band"
lo, hi = m - k_warn * sigma, m + k_warn * sigma
levels = [lv for _, _, lv in history]
if x_level > max(levels):
caveats.append(ph.caveat("record_high", ph.record_text(True, series.label, n, unit)))
elif x_level < min(levels):
caveats.append(ph.caveat("record_low", ph.record_text(False, series.label, n, unit)))
if series.nulls:
caveats.append(ph.caveat("null_values", ph.null_periods_text(series.nulls, series.label)))
current = ph.current_text(x, basis=basis, lag=lag, grain=series.grain)
if sigma == 0:
shown = ph.pct(m) if basis == "change" else ph.amount(m)
main = ph.baseline_flat_phrase(code, series.label, current, n, unit, per, shown)
else:
main = ph.baseline_phrase(code, series.label, current, n, unit, ph.band_text(lo, hi, basis=basis))
rows = [[p.raw, canon(lv), canon(h) if basis == "change" else None, "历史"] for p, h, lv in history]
rows.append([target.raw, canon(x_level), canon(x) if basis == "change" else None, "本期"])
table = {"columns": [series.period, "value", "change", "role"],
"labels": {series.period: column_label(env, series.snap, series.period), "value": series.label,
"change": "变化率", "role": "说明"},
"rows": rows}
values = {"x": canon(x), "value": canon(x_level), "median": canon(m), "band_lo": canon(lo), "band_hi": canon(hi),
"z": canon(z) if z is not None else None, "rank": str(stats.rank_desc(x, hv)), "n_history": str(n),
"hist_min": canon(min(hv)), "hist_max": canon(max(hv))}
verdict = {"code": code, "subjects": [], "direction": sign_word(x - m)}
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats)
# --------------------------------------------------------------------------
# 趋势
# --------------------------------------------------------------------------
def run_length(values: list[Fraction]) -> tuple[int, str | None]:
"""末尾连续同向:从最后一期往前数,相邻变化同号的有几个(变化为 0 断开)。"""
changes = [b - a for a, b in zip(values, values[1:])]
if not changes or changes[-1] == 0:
return 0, None
sign = changes[-1] > 0
count = 0
for c in reversed(changes):
if c == 0 or (c > 0) != sign:
break
count += 1
return count, "up" if sign else "down"
def trend(raw: Mapping[str, Any], env: Env) -> Any:
used = ["trend_min_points", "trend_alpha", "max_history"]
loaded = _load(raw, env, used)
if not isinstance(loaded, Series):
return loaded
series, params = loaded, loaded.params
unit, per = pd.COUNT_WORD[series.grain], pd.PER_WORD[series.grain]
minimum = int(env.settings["trend_min_points"])
criterion = _criterion(env, None, unit)
if not series.points:
return refuse(env, params, used, "history_short", f"只有 0 期,至少要 {minimum} 期", criterion=criterion)
last = series.points[-1][0]
caveats: list[dict[str, str]] = []
status, why = _completeness(env, series, last)
if status == "short":
return refuse(env, params, used, "partial_target", f"{why},和完整的{per}比没有意义", criterion=criterion)
if status == "unknown" and last.start is not None:
caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE))
cap = int(env.settings["max_history"]) + 1
tail = [series.points[-1]]
for point in reversed(series.points[:-1]):
if len(tail) >= cap or point[0].ordinal != tail[-1][0].ordinal - 1:
break
tail.append(point)
tail.reverse()
n = len(tail)
criterion = _criterion(env, n, unit)
if n < minimum:
gap = len(series.points) > n and len(tail) < cap
if gap:
return refuse(env, params, used, "series_gap",
f"最后连着的只有 {n} 期,再往前缺期(至少要连着的 {minimum} 期)", criterion=criterion)
return refuse(env, params, used, "history_short", f"只有 {n} 期,至少要 {minimum} 期", criterion=criterion)
ys = [v for _, v in tail]
xs = [Fraction(i) for i in range(n)]
slope = stats.theil_sen(xs, ys)
med = stats.median(ys)
rel = slope / abs(med) if med != 0 else None
mk = stats.mann_kendall(ys)
alpha = env.threshold("trend_alpha")
p = mk.p if isinstance(mk.p, Fraction) else to_fraction(mk.p)
if p is not None and p < alpha and slope > 0:
code = "rising"
elif p is not None and p < alpha and slope < 0:
code = "falling"
else:
code = "no_trend"
run, direction = run_length(ys)
if series.grain == "day":
caveats.append(ph.caveat("weekday_effect", ph.WEEKDAY_EFFECT))
if series.nulls:
caveats.append(ph.caveat("null_values", ph.null_periods_text(series.nulls, series.label)))
slope_text = ph.pct(rel) if rel is not None else ph.amount(slope, signed=True)
main = ph.trend_phrase(code, series.label, n, unit, per, slope_text,
(run, direction) if run >= 3 and direction else None) # type: ignore[arg-type]
rows = [[p_.raw, canon(v), canon(v - tail[i - 1][1]) if i else None] for i, (p_, v) in enumerate(tail)]
table = {"columns": [series.period, "value", "change"],
"labels": {series.period: column_label(env, series.snap, series.period), "value": series.label,
"change": "比上一期"},
"rows": rows}
values = {"slope": canon(slope), "rel_slope": canon(rel) if rel is not None else None, "p": canon(mk.p),
"s": str(mk.s), "n": str(n), "run_length": str(run), "run_direction": direction}
verdict = {"code": code, "subjects": [], "direction": "up" if code == "rising" else "down" if code == "falling"
else None}
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats)
__all__ = ["MAD_SCALE", "baseline", "run_length", "trend"]
+467
View File
@@ -0,0 +1,467 @@
"""分析用的统计函数(docs/design/analysis-round/01-算子与判定.md §3–§5,总方案 D13):纯 Python,不加依赖、不用 numpy。
为什么不装 scipy:几十 MB 的依赖只为几个函数,本机装包慢;评测本来就有一套纯 Python 的统计(Wilson 区间、精确 McNemar),
两边要是同一套。所以 Wilson 从 eval/stats 搬到这里,评测改为引用(行为不变)。每个函数都用公认的表值钉住
(tests/test_analysis_stats.py:正态 0.975 分位 1.959964、t(10) 双侧 0.05 临界值 2.228139、Fisher 品茶表双侧 0.4857……)。
- 正态:分布函数用 math.erfc;分位数用 Acklam 的有理逼近再按分布函数做两步 Halley 修正(到浮点精度)。
- t 分布:分布函数用正则化不完全 β 函数(连分式,Lentz 法);自由度很大(> 1e7)时和正态差在 1e-7 以内,直接按正态算;
分位数按分布函数二分求逆。
- 比率:Wilson 区间(缺省按评测的写法四舍五入到 4 位,places=None 不取整)、两比率之差的 Newcombe 区间(混合 score
法,即 Newcombe 1998 的方法 10)、合并比例的 z 检验、Fisher 精确检验(超几何概率用 math.lgamma 累加,双侧按「不比
观测到的那张表更可能」的表求和,比较时留 1e-7 的相对余量,同 R 和 scipy)。
- 均值:Welch t 检验,自由度按 Welch–Satterthwaite。
- 多重比较:Holm 逐步校正。
- 趋势:Mann–Kendall(n ≤ 10 且没有并列时用精确分布:S 的分布由逆序数的分布递推算出,不查表;否则正态近似,方差按
并列修正,带连续性校正)、Theil–Sen 斜率(两两斜率的中位数)。
- 中位数、中位绝对偏差、名次。
能精确算的照精确算:中位数、MAD、Theil–Sen 对 Fraction 输入给 Fraction,Mann–Kendall 的精确 p 是 Fraction。浮点只出现在
要开方、取对数、查分布的地方,结论里的数再按 analysis/numbers 统一成规范的十进制字符串。
"""
from __future__ import annotations
import math
from collections import Counter
from collections.abc import Sequence
from fractions import Fraction
from typing import Any, NamedTuple
#: 95% 区间的 z(正态 0.975 分位)
Z95 = 1.959963984540054
_SQRT2 = math.sqrt(2.0)
#: t 分布自由度大到这个数以上时按正态算(相对误差约 1/自由度)
_T_AS_NORMAL_DF = 1e7
# --------------------------------------------------------------------------
# 正态分布
# --------------------------------------------------------------------------
def normal_cdf(x: float) -> float:
"""标准正态分布函数 Φ(x)。"""
return 0.5 * math.erfc(-x / _SQRT2)
def normal_two_sided(z: float) -> float:
"""标准正态的双侧尾概率 P(|Z| ≥ |z|)。"""
return math.erfc(abs(z) / _SQRT2)
_A = (-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, 1.383577518672690e+02,
-3.066479806614716e+01, 2.506628277459239e+00)
_B = (-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, 6.680131188771972e+01,
-1.328068155288572e+01)
_C = (-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, -2.549732539343734e+00,
4.374664141464968e+00, 2.938163982698783e+00)
_D = (7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, 3.754408661907416e+00)
def normal_ppf(p: float) -> float:
"""标准正态的 p 分位数 Φ⁻¹(p)。p ≤ 0 为负无穷,p ≥ 1 为正无穷。"""
if p <= 0.0:
return -math.inf
if p >= 1.0:
return math.inf
low = 0.02425
if p < low:
q = math.sqrt(-2 * math.log(p))
x = (((((_C[0] * q + _C[1]) * q + _C[2]) * q + _C[3]) * q + _C[4]) * q + _C[5]) / \
((((_D[0] * q + _D[1]) * q + _D[2]) * q + _D[3]) * q + 1)
elif p <= 1 - low:
q = p - 0.5
r = q * q
x = (((((_A[0] * r + _A[1]) * r + _A[2]) * r + _A[3]) * r + _A[4]) * r + _A[5]) * q / \
(((((_B[0] * r + _B[1]) * r + _B[2]) * r + _B[3]) * r + _B[4]) * r + 1)
else:
q = math.sqrt(-2 * math.log1p(-p))
x = -(((((_C[0] * q + _C[1]) * q + _C[2]) * q + _C[3]) * q + _C[4]) * q + _C[5]) / \
((((_D[0] * q + _D[1]) * q + _D[2]) * q + _D[3]) * q + 1)
for _ in range(2):
# Halley 修正:误差按分布函数现算,两步就到浮点精度
e = (normal_cdf(x) - p) if p < 0.5 else -(0.5 * math.erfc(x / _SQRT2) - (1 - p))
u = e * math.sqrt(2 * math.pi) * math.exp(x * x / 2)
x = x - u / (1 + x * u / 2)
return x
# --------------------------------------------------------------------------
# 正则化不完全 β 函数与 t 分布
# --------------------------------------------------------------------------
_CF_MAX_ITER = 20_000
_CF_EPS = 1e-15
_CF_TINY = 1e-300
def _beta_cf(a: float, b: float, x: float) -> float:
"""不完全 β 函数的连分式(Lentz 法,Numerical Recipes 的 betacf)。"""
qab, qap, qam = a + b, a + 1.0, a - 1.0
c = 1.0
d = 1.0 - qab * x / qap
d = 1.0 / (d if abs(d) >= _CF_TINY else _CF_TINY)
h = d
for m in range(1, _CF_MAX_ITER + 1):
m2 = 2 * m
aa = m * (b - m) * x / ((qam + m2) * (a + m2))
d = 1.0 + aa * d
d = 1.0 / (d if abs(d) >= _CF_TINY else _CF_TINY)
c = 1.0 + aa / c
c = c if abs(c) >= _CF_TINY else _CF_TINY
h *= d * c
aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2))
d = 1.0 + aa * d
d = 1.0 / (d if abs(d) >= _CF_TINY else _CF_TINY)
c = 1.0 + aa / c
c = c if abs(c) >= _CF_TINY else _CF_TINY
step = d * c
h *= step
if abs(step - 1.0) < _CF_EPS:
break
return h
def betainc(a: float, b: float, x: float) -> float:
"""正则化不完全 β 函数 I_x(a, b),a、b > 0,0 ≤ x ≤ 1。"""
if x <= 0.0:
return 0.0
if x >= 1.0:
return 1.0
log_front = math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b) + a * math.log(x) + b * math.log1p(-x)
if x < (a + 1.0) / (a + b + 2.0):
return math.exp(log_front) * _beta_cf(a, b, x) / a
return 1.0 - math.exp(log_front) * _beta_cf(b, a, 1.0 - x) / b
def t_two_sided(t: float, df: float) -> float:
"""自由度 df 的 t 分布的双侧尾概率 P(|T| ≥ |t|)。"""
if math.isinf(t):
return 0.0
if df > _T_AS_NORMAL_DF or math.isinf(df):
return normal_two_sided(t)
return min(1.0, betainc(df / 2.0, 0.5, df / (df + t * t)))
def t_cdf(t: float, df: float) -> float:
"""自由度 df 的 t 分布的分布函数。"""
tail = t_two_sided(t, df) / 2.0
return 1.0 - tail if t >= 0 else tail
def t_ppf(p: float, df: float) -> float:
"""自由度 df 的 t 分布的 p 分位数:按分布函数二分求逆。"""
if p <= 0.0:
return -math.inf
if p >= 1.0:
return math.inf
if p == 0.5:
return 0.0
if df > _T_AS_NORMAL_DF or math.isinf(df):
return normal_ppf(p)
target = min(p, 1.0 - p)
lo, hi = -1.0, 0.0
while t_cdf(lo, df) > target:
lo, hi = lo * 2.0, lo
if lo < -1e12:
break
for _ in range(200):
mid = (lo + hi) / 2.0
if t_cdf(mid, df) > target:
hi = mid
else:
lo = mid
if hi - lo <= 1e-15 * max(1.0, abs(mid)):
break
x = (lo + hi) / 2.0
return x if p < 0.5 else -x
# --------------------------------------------------------------------------
# 比率
# --------------------------------------------------------------------------
def wilson(passed: int, n: int, z: float = Z95, *, places: int | None = 4) -> tuple[float, float] | None:
"""Wilson 区间(缺省 95%)。n 为 0 时没有区间。places 是四舍五入的位数(缺省 4 位,评测一直这样显示);None 不取整。"""
if n <= 0:
return None
p = passed / n
denom = 1 + z * z / n
centre = (p + z * z / (2 * n)) / denom
half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom
low, high = max(0.0, centre - half), min(1.0, centre + half)
if places is None:
return low, high
return round(low, places), round(high, places)
def newcombe(x1: int, n1: int, x2: int, n2: int, z: float = Z95) -> tuple[float, float] | None:
"""两比率之差 p1 − p2 的 Newcombe 区间(混合 score 法:两边各自的 Wilson 区间合成)。有一边样本为 0 时没有区间。"""
w1, w2 = wilson(x1, n1, z, places=None), wilson(x2, n2, z, places=None)
if w1 is None or w2 is None:
return None
p1, p2 = x1 / n1, x2 / n2
(l1, u1), (l2, u2) = w1, w2
diff = p1 - p2
delta = math.sqrt((p1 - l1) ** 2 + (u2 - p2) ** 2)
epsilon = math.sqrt((u1 - p1) ** 2 + (p2 - l2) ** 2)
return diff - delta, diff + epsilon
def pooled_z(x1: int, n1: int, x2: int, n2: int) -> tuple[float, float] | None:
"""两比率相等的合并比例 z 检验:(z, 双侧 p)。有一边样本为 0 时为 None;合并比例是 0 或 1(两边全是或全不是)时 z 为 0、
p 为 1。"""
if n1 <= 0 or n2 <= 0:
return None
pooled = (x1 + x2) / (n1 + n2)
se = math.sqrt(pooled * (1 - pooled) * (1 / n1 + 1 / n2))
if se == 0:
return 0.0, 1.0
z = (x1 / n1 - x2 / n2) / se
return z, normal_two_sided(z)
def min_expected(x1: int, n1: int, x2: int, n2: int) -> float:
"""2×2 表(成功、失败 × 两组)各格的期望频数里最小的那个。"""
total = n1 + n2
if total <= 0:
return 0.0
successes, failures = x1 + x2, total - x1 - x2
return min(r * c / total for r in (n1, n2) for c in (successes, failures))
def _log_comb(n: int, k: int) -> float:
return math.lgamma(n + 1) - math.lgamma(k + 1) - math.lgamma(n - k + 1)
def fisher_exact(a: int, b: int, c: int, d: int) -> float:
"""2×2 表 [[a, b], [c, d]] 的 Fisher 精确检验,双侧 p:边际固定时,不比观测到的表更可能出现的那些表的概率之和。"""
r1, r2, c1 = a + b, c + d, a + c
n = r1 + r2
if n == 0:
return 1.0
lo, hi = max(0, c1 - r2), min(r1, c1)
base = _log_comb(n, c1)
def log_p(x: int) -> float:
return _log_comb(r1, x) + _log_comb(r2, c1 - x) - base
observed = log_p(a) + 1e-7 # 相对余量 1 + 1e-7(对数里约等于加 1e-7),同 R 和 scipy
total = sum(math.exp(lp) for lp in (log_p(x) for x in range(lo, hi + 1)) if lp <= observed)
return min(1.0, total)
# --------------------------------------------------------------------------
# 均值
# --------------------------------------------------------------------------
class Welch(NamedTuple):
t: float
df: float
p: float
def welch(m1: float, v1: float, n1: int, m2: float, v2: float, n2: int) -> Welch | None:
"""两组均值相等的 Welch t 检验。v 是样本方差(n − 1 作分母)。有一组样本不到 2 个时为 None(方差算不了)。两组方差都是 0:
均值相同 p 为 1,不同 p 为 0。"""
if n1 < 2 or n2 < 2:
return None
s1, s2 = v1 / n1, v2 / n2
se2 = s1 + s2
if se2 <= 0:
return Welch(0.0, math.inf, 1.0) if m1 == m2 else Welch(math.copysign(math.inf, m1 - m2), math.inf, 0.0)
t = (m1 - m2) / math.sqrt(se2)
denom = (s1 * s1 / (n1 - 1) if s1 else 0.0) + (s2 * s2 / (n2 - 1) if s2 else 0.0)
df = se2 * se2 / denom if denom > 0 else math.inf
return Welch(t, df, t_two_sided(t, df))
def welch_interval(m1: float, v1: float, n1: int, m2: float, v2: float, n2: int,
level: float = 0.95) -> tuple[float, float] | None:
"""m1 − m2 的 Welch 区间(缺省 95%)。算不了检验时也没有区间。"""
test = welch(m1, v1, n1, m2, v2, n2)
if test is None:
return None
se = math.sqrt(v1 / n1 + v2 / n2)
if se == 0:
return m1 - m2, m1 - m2
q = t_ppf(0.5 + level / 2.0, test.df)
return m1 - m2 - q * se, m1 - m2 + q * se
# --------------------------------------------------------------------------
# 多重比较
# --------------------------------------------------------------------------
def holm(pvalues: Sequence[float | None]) -> list[float | None]:
"""Holm 逐步校正:按原始 p 从小到大,第 i 个(从 1 数)乘以 m − i + 1,再取到它为止的最大值,不超过 1。None(这一对算不了)
不计入次数、原样留着。"""
present = sorted((p, i) for i, p in enumerate(pvalues) if p is not None)
m = len(present)
out: list[float | None] = [None] * len(pvalues)
running = 0.0
for rank, (p, i) in enumerate(present):
running = max(running, min(1.0, (m - rank) * p))
out[i] = running
return out
# --------------------------------------------------------------------------
# 趋势
# --------------------------------------------------------------------------
def _sign(x: Any) -> int:
return (x > 0) - (x < 0)
class MannKendall(NamedTuple):
s: int
var: float # S 的方差(并列修正后)
z: float | None # 正态近似的 z(精确分布时为 None)
p: float | Fraction # 双侧 p;精确分布时是 Fraction
exact: bool
#: 精确分布最多到几期:再多 n! 太大,正态近似已经足够
EXACT_MAX_N = 10
def _inversion_counts(n: int) -> list[int]:
"""1..n 的排列里,逆序数为 k 的排列有几个(k = 0 … n(n−1)/2)。递推:第 m 个数插进 m 个位置,逆序数各加 0 … m−1。"""
counts = [1]
for m in range(2, n + 1):
nxt = [0] * (len(counts) + m - 1)
for k, c in enumerate(counts):
for j in range(m):
nxt[k + j] += c
counts = nxt
return counts
def _scaled(values: Sequence[Any]) -> tuple[list[int], int] | None:
"""一组精确的数(整数、Fraction)乘上公分母换成整数:(整数, 公分母)。有浮点的为 None。两两相减、比大小在整数上做,
比 Fraction 快得多,结果一样。"""
if not all(isinstance(v, (int, Fraction)) and not isinstance(v, bool) for v in values):
return None
scale = math.lcm(*(Fraction(v).denominator for v in values)) if values else 1
return [int(Fraction(v) * scale) for v in values], scale
def mann_kendall(values: Sequence[Any]) -> MannKendall:
"""Mann–Kendall 趋势检验:S = Σ_{i<j} sgn(y_j − y_i),双侧 p。n ≤ EXACT_MAX_N 且没有并列时用精确分布(没有并列时
S = N − 2·逆序数,N = n(n−1)/2,逆序数的分布递推算出);否则正态近似:Var(S) = [n(n−1)(2n+5) − Σt(t−1)(2t+5)] / 18
(t 是每组并列的个数),z = (S − sgn S) / √Var(连续性校正)。"""
n = len(values)
scaled = _scaled(values)
ys = scaled[0] if scaled is not None else list(values)
s = sum(_sign(ys[j] - ys[i]) for i in range(n) for j in range(i + 1, n))
ties = [t for t in Counter(ys).values() if t > 1]
var = (n * (n - 1) * (2 * n + 5) - sum(t * (t - 1) * (2 * t + 5) for t in ties)) / 18.0
if n < 2:
return MannKendall(0, 0.0, None, Fraction(1), True)
if not ties and n <= EXACT_MAX_N:
counts = _inversion_counts(n)
pairs = n * (n - 1) // 2
hit = sum(c for k, c in enumerate(counts) if abs(pairs - 2 * k) >= abs(s))
return MannKendall(s, var, None, Fraction(hit, math.factorial(n)), True)
if var <= 0:
return MannKendall(s, var, 0.0, 1.0, False)
z = (s - _sign(s)) / math.sqrt(var)
return MannKendall(s, var, z, normal_two_sided(z), False)
def theil_sen(xs: Sequence[Any], ys: Sequence[Any]) -> Any:
"""Theil–Sen 斜率:所有两点(x 不同)连线斜率的中位数。不到两个点为 None。xs、ys 都是精确的数时结果是 Fraction:斜率
在公分母上写成整数之比,按 int / int(正确舍入、单调)的浮点排,只把浮点相等的那几个按精确值比,n = 261 的三万多个
斜率也不逐个造 Fraction。"""
sx, sy = _scaled(xs), _scaled(ys)
if sx is None or sy is None:
slopes = [(ys[j] - ys[i]) / (xs[j] - xs[i]) for i in range(len(xs)) for j in range(i + 1, len(xs))
if xs[j] != xs[i]]
return median(slopes) if slopes else None
(X, dx), (Y, dy) = sx, sy
pairs = []
for i in range(len(X)):
for j in range(i + 1, len(X)):
den = (X[j] - X[i]) * dy
if den:
num = (Y[j] - Y[i]) * dx
pairs.append((num, den) if den > 0 else (-num, -den))
if not pairs:
return None
keys = [num / den for num, den in pairs]
order = sorted(range(len(pairs)), key=keys.__getitem__)
def kth(k: int) -> Fraction:
pivot = keys[order[k]]
lo, hi = k, k + 1
while lo > 0 and keys[order[lo - 1]] == pivot:
lo -= 1
while hi < len(order) and keys[order[hi]] == pivot:
hi += 1
block = sorted(Fraction(*pairs[t]) for t in order[lo:hi])
return block[k - lo]
m = len(pairs)
return kth(m // 2) if m % 2 else (kth(m // 2 - 1) + kth(m // 2)) / 2
# --------------------------------------------------------------------------
# 位置与离散
# --------------------------------------------------------------------------
#: 超过这么多个值时中位数先按浮点排(Theil–Sen 的两两斜率有几万个,逐个按 Fraction 比太慢)
_FAST_MEDIAN = 512
def _kth(values: Sequence[Any], k: int) -> Any:
"""从小到大第 k 个(从 0 数),精确:Fraction 换成浮点是正确舍入的、单调的——浮点小的精确值一定小,排错的只可能是浮点
相等的那几个,所以先按浮点排,再只把和第 k 个浮点相等的那一段按精确值排。"""
keyed = sorted((float(v), i) for i, v in enumerate(values))
pivot = keyed[k][0]
lo, hi = k, k + 1
while lo > 0 and keyed[lo - 1][0] == pivot:
lo -= 1
while hi < len(keyed) and keyed[hi][0] == pivot:
hi += 1
block = sorted(values[i] for _, i in keyed[lo:hi])
return block[k - lo]
def median(values: Sequence[Any]) -> Any:
"""中位数(偶数个时取中间两个的平均)。Fraction 进 Fraction 出。没有值为 None。"""
n = len(values)
if n == 0:
return None
mid = n // 2
if n > _FAST_MEDIAN:
upper = _kth(values, mid)
return upper if n % 2 else (_kth(values, mid - 1) + upper) / 2
data = sorted(values)
if n % 2:
return data[mid]
return (data[mid - 1] + data[mid]) / 2
def mad(values: Sequence[Any], center: Any | None = None) -> Any:
"""中位绝对偏差:各值离中位数(或给定的中心)的绝对距离的中位数。"""
if not values:
return None
c = median(values) if center is None else center
return median([abs(v - c) for v in values])
def rank_desc(x: Any, others: Sequence[Any]) -> int:
"""x 在「others + x」里从大到小的名次(并列取最好的名次:比它大的有几个再加 1)。"""
return 1 + sum(1 for o in others if o > x)
__all__ = [
"EXACT_MAX_N", "MannKendall", "Welch", "Z95", "betainc", "fisher_exact", "holm", "mad", "mann_kendall", "median",
"min_expected", "newcombe", "normal_cdf", "normal_ppf", "normal_two_sided", "pooled_z", "rank_desc", "t_cdf",
"t_ppf", "t_two_sided", "theil_sen", "welch", "welch_interval", "wilson",
]
+91 -3
View File
@@ -447,9 +447,14 @@ async def _fallback(flt: _Filter, facts: Facts) -> _Range | None:
found = await facts(flt.table_name)
except Exception: # noqa: BLE001
return None
return _facts_range(found, flt.column_name)
def _facts_range(found: Any, column: str) -> _Range | None:
"""剖析事实(last_facts 的一项)里这一列的范围:先剖析的 date_ranges,再取值观察的最新日期。"""
if not isinstance(found, Mapping):
return None
wanted = name_key(flt.column_name)
wanted = name_key(column)
for item in found.get("date_ranges") or []:
if isinstance(item, Mapping) and name_key(str(item.get("column") or "")) == wanted:
lo, hi = _as_day(item.get("min")), _as_day(item.get("max"))
@@ -806,5 +811,88 @@ def no_data_of(snapshot: Any, column: Any) -> dict[str, Any] | None:
return None
__all__ = ["BUDGET_S", "Cache", "NO_DATA_SHORT", "PROBE_TIMEOUT_S", "Window", "filters", "narrowed", "no_data_of",
"result_gaps", "sql_gaps", "zero_columns"]
# ==========================================================================
# 给分析的可比性检查用的几个小函数(app/analysis/comparable.py、probe.py:窗口长度、整期)。读法、写法、探查都和上面同一套,
# 只是把中间结果交出去:分析结论要把现查的范围、在库里算出的界记下来,复算时用记下的值
# ==========================================================================
def parse_sql(sql: Any, read: str | None) -> Any:
"""一条查询解析成语法树(同覆盖检查:太长、带模板占位、不止一条、不是查询的为 None)。"""
return _parse(sql, read)
def as_day(value: Any, kind: str | None = None) -> _Day | None:
"""一个值归一成日期(DATE / DATETIME、'YYYY-MM-DD'、'YYYYMMDD'、8 位整数);认不出为 None。"""
return _as_day(value, kind)
def literal_bound(node: Any) -> tuple[bool, _Day | None]:
"""时间条件的一个界:(是不是字面量, 字面量归一成的日期)。不是字面量的是按运行当天现算的,要在库里算。"""
return (True, _literal_day(node)) if _is_literal(node) else (False, None)
def window_of(conds: list[tuple[str, Any, bool]], days: Mapping[int, _Day | None]) -> Window | None:
"""几个条件合成一段区间(同 _window:下界取最晚、上界取最早,「>」「<」的界不带时刻时按天挪一天)。days 是
{id(界的节点): 日期}。不核对列和界的写法比不比得起来:分析结论里表的范围只记了日期。"""
return _window(conds, days, _Range(None, None, "none"), None)
def _range_out(data: _Range | None) -> dict[str, Any] | None:
if data is None:
return None
return {"min": data.low.day.isoformat() if data.low else None,
"max": data.high.day.isoformat() if data.high else None,
"basis": data.basis, "at": data.day, "empty": data.empty}
def facts_range(found: Any, column: str) -> dict[str, Any] | None:
"""剖析事实(catalog_profile_jobs.last_facts 的一项)里这一列的范围,形状同 column_range;没有为 None。"""
return _range_out(_facts_range(found, column))
async def column_range(source: Any, table: str, column: str, *, facts: Facts | None = None,
cache: Cache | None = None) -> dict[str, Any] | None:
"""这一列在这张表里的实际范围:现查 MIN / MAX(只读守卫、PROBE_TIMEOUT_S 秒、同一张表 LIVE_TTL_S 秒内不重查),查不成
退回剖析、取值观察(facts)。{min, max, basis, at, empty}:min / max 是 ISO 日期(不知道的那一头为 None),basis 是 live /
profile / observed,at 是剖析、观察的日期(现查为 None),表里这一列一个值都没有时 empty 为真。都没有为 None。从不抛异常。"""
from sqlglot import exp
try:
parts = [p for p in str(table).split(".") if p]
node = exp.Table(this=exp.to_identifier(parts[-1]),
**({"db": exp.to_identifier(parts[-2])} if len(parts) > 1 else {}))
flt = _Filter((), node, str(table), exp.column(str(column)), str(column))
return _range_out(await _range(source, flt, _read(getattr(source, "kind", None)), cache or Cache(), facts))
except Exception: # noqa: BLE001 - 探查失败就当不知道
logger.info("可比性检查的范围探查没有成功(数据源 %s)", getattr(source, "name", ""), exc_info=True)
return None
async def bound_values(source: Any, texts: list[str]) -> list[Any] | None:
"""按运行当天现算的几个界(SQL 表达式,按这个数据源的方言写)在库里算出来:一条 SELECT 界1, 界2 …(Oracle 加 FROM
DUAL),走只读守卫和 PROBE_TIMEOUT_S。交回各个值(日期、时刻按 ISO 写,其余照库里交回的);查不成为 None。从不抛异常。"""
if not texts:
return []
read = _read(getattr(source, "kind", None))
try:
found = await _probe(source, _select_sql([f"{t} AS b{i}" for i, t in enumerate(texts)], read))
except Exception: # noqa: BLE001
return None
if found is None:
return None
values = list(found[0]) + [None] * len(texts)
out: list[Any] = []
for value in values[:len(texts)]:
if isinstance(value, datetime):
out.append(value.isoformat(sep=" ", timespec="seconds"))
elif isinstance(value, date):
out.append(value.isoformat())
else:
out.append(value)
return out
__all__ = ["BUDGET_S", "Cache", "NO_DATA_SHORT", "PROBE_TIMEOUT_S", "Window", "as_day", "bound_values", "column_range",
"facts_range", "filters", "literal_bound", "narrowed", "no_data_of", "parse_sql", "result_gaps", "sql_gaps",
"window_of", "zero_columns"]
+3 -13
View File
@@ -22,25 +22,15 @@ from collections import Counter
from collections.abc import Iterable, Mapping, Sequence
from typing import Any
#: 95% 区间的 z
Z95 = 1.959963984540054
#: Wilson 区间和 95% 区间的 z 在分析的统计里(总方案 D13:分析和评测共用一套纯 Python 的统计),这里引用、行为不变
from app.analysis.stats import Z95, wilson
#: 问数据页等复核最多这么久(毫秒,同 answer.REVIEW_PAGE_WAIT_S)
REVIEW_PAGE_WAIT_MS = 25_000
TIERS = ("formal", "degraded", "withheld", "explore")
TIMING_KEYS = ("build_ms", "run_wall_ms", "run_active_ms", "review_ms", "total_ms")
def wilson(passed: int, n: int, z: float = Z95) -> tuple[float, float] | None:
"""Wilson 区间(缺省 95%)。n 为 0 时没有区间。"""
if n <= 0:
return None
p = passed / n
denom = 1 + z * z / n
centre = (p + z * z / (2 * n)) / denom
half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom
return round(max(0.0, centre - half), 4), round(min(1.0, centre + half), 4)
def mcnemar_exact(only_a: int, only_b: int) -> float:
"""精确 McNemar 双侧 p 值:不一致的那几题里,「只 A 对」服从 Binomial(b + c, 0.5)。没有不一致的题时为 1。"""
n = only_a + only_b
+127
View File
@@ -0,0 +1,127 @@
"""分析算子测试用的样例:虚构零售库 shop(地区华东、华南、华北、西南,渠道线上、线下,销售额、订单数、退款),表名、列名、
数字都是编的。analyze 跑一次 analysis.run,顺手用 recompute 复算一遍、核对逐字段相同(contract.same),所以每个用例都同时
钉住「复算和运行逐字段相同」。"""
from __future__ import annotations
import json
from collections.abc import Mapping, Sequence
from datetime import date
from typing import Any
from app import analysis
from app.analysis.contract import CallError, effective_settings, same
from app.core.artifact_store import canonical_json, content_hash
#: 测试里的「运行当天」
TODAY = date(2026, 10, 21)
def snap(ref: str, sql: str, columns: Sequence[str], rows: Sequence[Sequence[Any]], *,
types: Mapping[str, str] | None = None, source: str = "shop", truncated: bool = False,
masked: Sequence[str] = (), **extra: Any) -> dict[str, Any]:
"""一份输入快照(InputSnapshot 的形状)。artifact 按内容算,复算按它取回。"""
body = {"source": source, "sql": sql, "columns": list(columns), "rows": [list(r) for r in rows],
"truncated": truncated, **({"column_types": dict(types)} if types else {}), **extra}
return {"ref": ref, "artifact": content_hash(canonical_json(body)), "node_id": "q1", "masked": list(masked), **body}
def item(value: Any, status: str = "confirmed", source: str = "human") -> dict[str, Any]:
return {"value": value, "source": source, "status": status}
def notes(*, columns: Mapping[str, Mapping[str, Any]] | None = None, business_date: str | None = None) -> dict[str, Any]:
"""一张表的目录 notes:columns 是 {列: {label / measure: 值或 item(...)}}。"""
out: dict[str, Any] = {}
if columns:
out["columns"] = {c: {k: v if isinstance(v, Mapping) else item(v) for k, v in spec.items()}
for c, spec in columns.items()}
if business_date:
out["business_date"] = item({"column": business_date, "rule": "下单日", "timezone": "Asia/Shanghai"})
return out
#: orders 表的目录:地区、渠道、实收金额(流量)、退款单数
ORDERS = notes(columns={"region": {"label": "地区"}, "channel": {"label": "渠道"},
"amount": {"label": "销售额", "measure": "flow"},
"refund_flag": {"label": "退款单数", "measure": "flow"},
"order_date": {"label": "下单日"}},
business_date="order_date")
CATALOGS = {"shop": {"orders": ORDERS}}
async def analyze(op: str, mode: str | None = None, *, snaps: Sequence[Mapping[str, Any]], today: date = TODAY,
catalogs: Mapping[str, Any] | None = None, probe: Any = None, facts: Any = None,
parents: Mapping[str, Mapping[str, Any]] | None = None, settings: Mapping[str, Any] | None = None,
hypothesis: str | None = None, check: bool = True, **params: Any) -> dict[str, Any]:
"""跑一次 analysis.run;check 时再复算一遍,核对和封存的逐字段相同(hypothesis 除外)。"""
by_ref = {s["ref"]: s for s in snaps}
request: dict[str, Any] = {"op": op, "mode": mode, "params": params}
if hypothesis is not None:
request["hypothesis"] = hypothesis
context: dict[str, Any] = {"today": today, "catalogs": CATALOGS if catalogs is None else catalogs}
if probe is not None:
context["probe"] = probe
if facts is not None:
context["facts"] = facts
result = await analysis.run(request, by_ref.get, settings=effective_settings(settings), context=context,
parents=(parents or {}).get)
if check:
again = recomputed(result, snaps, parents)
assert same(result, again), _diff(result, again)
assert again["hypothesis"] == result["hypothesis"]
return dict(result)
def recomputed(result: Mapping[str, Any], snaps: Sequence[Mapping[str, Any]],
parents: Mapping[str, Mapping[str, Any]] | None = None) -> dict[str, Any]:
"""按结论里记下的复算:快照按 artifact 取回(只交快照内容本身,不交入口加的出处),上一层结论按内容哈希取回。"""
by_artifact = {s["artifact"]: {k: v for k, v in s.items() if k not in ("ref", "artifact", "node_id", "masked")}
for s in snaps}
by_hash = {content_hash(canonical_json(r)): r for r in (parents or {}).values()}
return dict(analysis.recompute(result, by_artifact.get, load_result=by_hash.get)) # type: ignore[arg-type]
def _diff(a: Mapping[str, Any], b: Mapping[str, Any]) -> str:
keys = sorted(k for k in set(a) | set(b) if a.get(k) != b.get(k))
return ";".join(f"{k}: {json.dumps(a.get(k), ensure_ascii=False)} ≠ {json.dumps(b.get(k), ensure_ascii=False)}"
for k in keys)
async def call_error(op: str, mode: str | None = None, **kwargs: Any) -> CallError:
"""跑一次,期望是调用错误。"""
try:
await analyze(op, mode, check=False, **kwargs)
except CallError as e:
return e
raise AssertionError("应当是调用错误")
def fixed_prober(ranges: Mapping[tuple[str, str], tuple[str | None, str | None]] | None = None, *,
bounds: Mapping[str, str] | None = None, at: str = "2026-10-21T09:00:00") -> Any:
"""测试用的探查器:ranges 是 {(表, 列): (最早, 最晚)},bounds 是 {界的 SQL: 日期};没写的查不到。"""
class Prober:
def __init__(self) -> None:
self.calls: list[tuple[str, ...]] = []
async def __call__(self, source: str, table: str, column: str) -> dict[str, Any] | None:
self.calls.append((source, table, column))
found = (ranges or {}).get((table, column))
if found is None:
return None
return {"source": source, "table": table, "column": column, "min": found[0], "max": found[1],
"basis": "live", "at": at}
async def bounds(self, source: str, expr: str) -> dict[str, Any] | None:
self.calls.append((source, expr))
value = (bounds or {}).get(expr)
return None if value is None else {"source": source, "expr": expr, "value": value, "basis": "live", "at": at}
def dialect(self, source: str) -> str | None:
return "sqlite"
return Prober()
__all__ = ["CATALOGS", "ORDERS", "TODAY", "analyze", "call_error", "fixed_prober", "item", "notes", "recomputed",
"snap"]
+182
View File
@@ -0,0 +1,182 @@
"""可比性检查(app/analysis/comparable.py,细案 01 §6):只差日期 / 差了别的筛选 / 结构不同、窗口不等(流量不可比、比率只提醒)、
本期不完整、周末天数不同、探查失败退回剖析事实、都没有就核对不了。每个用例都顺带核对复算和运行逐字段相同(复算用记下的
probes,不现查)。"""
from __future__ import annotations
from app.analysis import comparable_reasons
from tests.analysis_samples import analyze, fixed_prober, snap
WINDOW = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= '{a}' AND order_date < '{b}'"
RANGE = {("orders", "order_date"): ("2025-01-01", "2026-10-20")}
def period(ref, a, b, sql=WINDOW, **kw):
return snap(ref, sql.format(a=a, b=b), ["gmv"], [[100]], types={"gmv": "number"}, **kw)
async def compare(base, target, *, probe=None, **params):
params = {"base_query": base["ref"], "target_query": target["ref"], **params}
return await analyze("comparable", snaps=[base, target], probe=probe, **params)
def checks(result):
return {r[0]: r[1] for r in result["table"]["rows"]}
async def test_two_weeks_that_differ_only_in_dates_are_comparable():
result = await compare(period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19"),
probe=fixed_prober(RANGE), measure="gmv")
assert result["method"] == "comparable@1" and result["mode"] is None
assert result["verdict"] == {"code": "comparable", "subjects": [], "direction": None}
assert result["phrase"] == "两期可比"
assert checks(result) == {"同口径": "通过", "窗口长度": "通过", "整期": "通过", "星期构成": "通过", "数据问题": "通过"}
assert result["values"]["base_days"] == "7" and result["values"]["target_start"] == "2026-10-12"
assert result["values"]["window"] == "pass" and result["settings"] == {}
assert result["params"]["today"] == "2026-10-21" and result["params"]["measure"] == "gmv"
assert [p["column"] for p in result["probes"]] == ["order_date"]
assert result["table"]["columns"] == ["check", "status", "base", "target", "note"]
async def test_other_literals_differing_mean_two_segments_not_two_periods():
sql = WINDOW + " AND region = '{r}'"
base = snap("Q1", sql.format(a="2026-10-05", b="2026-10-12", r="华南"), ["gmv"], [[100]])
target = snap("Q2", sql.format(a="2026-10-12", b="2026-10-19", r="华东"), ["gmv"], [[100]])
result = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv")
assert result["verdict"]["code"] == "incomparable"
assert result["phrase"] == "两期不可比:两边除了日期,筛选的还不一样"
assert comparable_reasons(result) == ["两边除了日期,筛选的还不一样"]
async def test_a_different_structure_says_where_it_differs():
base = period("Q1", "2026-10-05", "2026-10-12")
target = period("Q2", "2026-10-12", "2026-10-19", sql=WINDOW + " AND status = 'paid'")
result = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv")
assert result["verdict"]["code"] == "incomparable"
assert result["phrase"] == "两期不可比:两份查询的口径结构不同(差在筛选条件)"
other = await compare(base, period("Q2", "2026-10-12", "2026-10-19", source="crm"), measure="gmv")
assert "两份查询来自不同的数据源" in other["phrase"]
async def test_unequal_windows_fail_for_flows_and_only_warn_otherwise():
base, target = period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-15")
flow = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv") # SUM(amount):目录标的流量
assert flow["verdict"]["code"] == "incomparable" and checks(flow)["窗口长度"] == "不通过"
assert "两段的天数不同:7 天对 3 天" in flow["phrase"]
avg_sql = "SELECT AVG(amount) AS aov FROM orders WHERE order_date >= '{a}' AND order_date < '{b}'"
ratio = await compare(period("Q1", "2026-10-05", "2026-10-12", sql=avg_sql),
period("Q2", "2026-10-12", "2026-10-15", sql=avg_sql), probe=fixed_prober(RANGE),
measure="gmv")
assert ratio["verdict"]["code"] == "conditional" and checks(ratio)["窗口长度"] == "提醒"
unlabeled = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv", catalogs={})
assert unlabeled["verdict"]["code"] == "conditional"
assert any(c["code"] == "measure_unlabeled" for c in unlabeled["caveats"])
async def test_whole_months_count_as_the_same_length():
result = await compare(period("Q1", "2026-08-01", "2026-09-01"), period("Q2", "2026-09-01", "2026-10-01"),
probe=fixed_prober(RANGE), measure="gmv")
assert result["verdict"]["code"] == "comparable" and result["values"]["base_days"] == "31"
async def test_an_unfinished_period_is_not_a_whole_period():
result = await compare(period("Q1", "2026-10-12", "2026-10-19"), period("Q2", "2026-10-19", "2026-10-26"),
probe=fixed_prober(RANGE), measure="gmv")
assert result["verdict"]["code"] == "incomparable"
assert result["phrase"] == "两期不可比:本期数据只到 2026-10-20,还差 5 天"
late = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-16")})
behind = await compare(period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19"),
probe=late, measure="gmv")
assert behind["phrase"] == "两期不可比:本期数据只到 2026-10-16,还差 2 天"
async def test_weekend_composition_of_short_windows():
# 周一到周三 对 周五到周日
result = await compare(period("Q1", "2026-10-05", "2026-10-08"), period("Q2", "2026-10-16", "2026-10-19"),
probe=fixed_prober(RANGE), measure="gmv")
assert result["verdict"]["code"] == "conditional"
assert result["phrase"] == "两期有条件可比:两段的周末天数不同:0 天对 2 天"
async def test_a_failed_probe_falls_back_to_profile_facts_then_says_it_cannot_check():
facts = {"shop": {"orders": {"at": "2026-10-20T03:00:00", "date_ranges": [
{"column": "order_date", "min": "2025-01-01", "max": "2026-10-19"}]}}}
base, target = period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19")
fallback = await compare(base, target, probe=fixed_prober({}), facts=facts, measure="gmv")
assert fallback["verdict"]["code"] == "comparable" and fallback["phrase"] == "两期可比"
assert fallback["probes"] == [{"source": "shop", "table": "orders", "column": "order_date", "min": "2025-01-01",
"max": "2026-10-19", "basis": "profile", "at": "2026-10-20"}]
nothing = await compare(base, target, measure="gmv")
assert nothing["verdict"]["code"] == "comparable"
assert nothing["phrase"] == "两期可比,整期核对不了" and nothing["probes"] == []
async def test_bounds_computed_from_the_run_day_are_evaluated_in_the_database_and_recorded():
sql = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', '{a}') AND order_date < date('now', '{b}')"
base = snap("Q1", sql.format(a="-16 days", b="-9 days"), ["gmv"], [[100]])
target = snap("Q2", sql.format(a="-9 days", b="-2 days"), ["gmv"], [[90]])
bounds = {"DATE('now', '-16 days')": "2026-10-05", "DATE('now', '-9 days')": "2026-10-12",
"DATE('now', '-2 days')": "2026-10-19"}
probe = fixed_prober(RANGE, bounds=bounds)
result = await compare(base, target, probe=probe, measure="gmv")
assert result["verdict"]["code"] == "comparable"
assert checks(result)["同口径"] == "通过" # 只差按运行当天现算的界
assert result["values"]["base_start"] == "2026-10-05" and result["values"]["target_end"] == "2026-10-18"
exprs = [p["expr"] for p in result["probes"] if "expr" in p]
assert exprs == ["DATE('now', '-16 days')", "DATE('now', '-9 days')", "DATE('now', '-2 days')"]
assert all(p["basis"] == "live" for p in result["probes"])
# 界算不出来:窗口核对不了,不判不可比
blind = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv")
assert blind["verdict"]["code"] == "comparable" and "窗口长度" in blind["phrase"]
async def test_data_problems_on_the_snapshots():
base = period("Q1", "2026-10-05", "2026-10-12")
empty = period("Q2", "2026-10-12", "2026-10-19", no_data=[{"columns": ["gmv"], "table": "orders",
"column": "order_date", "message": "没有数据"}])
result = await compare(base, empty, probe=fixed_prober(RANGE), measure="gmv")
assert result["verdict"]["code"] == "incomparable" and "本期查询的区间里这张表没有数据" in result["phrase"]
checked = period("Q2", "2026-10-12", "2026-10-19", checks=[{"code": "fanout_sum", "level": "error",
"message": "关联订单明细后对订单金额求和,会重复计算"}])
warned = await compare(base, checked, probe=fixed_prober(RANGE), measure="gmv")
assert warned["verdict"]["code"] == "conditional"
assert "本期查询的 SQL 检查有错误:关联订单明细后对订单金额求和,会重复计算" in warned["phrase"]
async def test_one_snapshot_grouped_by_period_is_naturally_the_same_query():
sql = "SELECT order_date AS day, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-01' GROUP BY 1"
rows = [[f"2026-10-{d:02d}", 100] for d in range(1, 20)]
s = snap("Q1", sql, ["day", "gmv"], rows, types={"day": "text", "gmv": "number"})
probe = fixed_prober(RANGE)
same_weekday = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="day", base="2026-10-05",
target="2026-10-12")
assert same_weekday["verdict"]["code"] == "comparable" and same_weekday["params"]["measure"] == "gmv"
assert same_weekday["params"]["grain"] == "day"
weekend = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="day", base="2026-10-09",
target="2026-10-10")
assert weekend["phrase"] == "两期有条件可比:两段的周末天数不同:0 天对 1 天"
# 期次有一截落在 SQL 的时间条件之外:基期被截掉了;本期(10 月)还没过完
monthly_rows = [["2026-09-01", 100], ["2026-10-01", 90]]
month = snap("Q1", "SELECT DATE(order_date, 'start of month') AS month, SUM(amount) AS gmv FROM orders "
"WHERE order_date >= '2026-09-05' GROUP BY 1", ["month", "gmv"], monthly_rows)
partial = await analyze("comparable", snaps=[month], probe=probe, query="Q1", period="month", base="2026-09-01",
target="2026-10-01", measure="gmv")
assert partial["verdict"]["code"] == "incomparable"
assert "基期数据从 2026-09-05 才开始,缺前 4 天" in partial["phrase"]
assert "本期数据只到 2026-10-20,还差 11 天" in partial["phrase"]
async def test_reasons_are_split_one_sentence_each():
result = await compare(period("Q1", "2026-10-05", "2026-10-08"), period("Q2", "2026-10-16", "2026-10-21"),
probe=fixed_prober(RANGE), measure="gmv")
reasons = comparable_reasons(result)
assert "两段的天数不同:3 天对 5 天" in reasons and "本期数据只到 2026-10-20,还差 0 天" not in reasons
assert all(";" not in r for r in reasons)
assert comparable_reasons({"table": None}) == []
async def test_a_masked_date_column_is_neither_used_nor_probed():
probe = fixed_prober(RANGE)
result = await compare(period("Q1", "2026-10-05", "2026-10-12", masked=["order_date"]),
period("Q2", "2026-10-12", "2026-10-19", masked=["order_date"]), probe=probe, measure="gmv")
assert probe.calls == [] and result["probes"] == []
assert result["verdict"]["code"] == "comparable" and "窗口长度" in result["phrase"]
+420
View File
@@ -0,0 +1,420 @@
"""变化拆解(app/analysis/decompose.py,细案 01 §2):三种恒等式逐位成立、辛普森、抵消、不闭合、下钻闭合与超深、新出现 /
消失的段、比率当可加的调用错误……每个用例都顺带核对复算和运行逐字段相同(tests/analysis_samples.analyze)。"""
from __future__ import annotations
import math
import random
from decimal import Decimal
from fractions import Fraction
import pytest
from app.analysis.decompose import lmdi, ratio_effects
from tests.analysis_samples import analyze, call_error, fixed_prober, notes, snap
SQL = "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week"
TYPES = {"region": "text", "week": "text", "gmv": "number"}
WEEKLY = [["华东", "2026-W41", 820], ["华东", "2026-W42", 700], ["华北", "2026-W41", 600], ["华北", "2026-W42", 560],
["华南", "2026-W41", 700], ["华南", "2026-W42", 666], ["西南", "2026-W41", 100], ["西南", "2026-W42", 100]]
def weekly(rows=WEEKLY, **kw):
return snap("Q2", SQL, ["region", "week", "gmv"], rows, types=TYPES, **kw)
async def additive(rows=WEEKLY, *, snaps=None, **params):
params = {"query": "Q2", "by": ["region"], "period": "week", "base": "2026-W41", "target": "2026-W42",
"measure": "gmv", **params}
return await analyze("decompose", "additive", snaps=snaps or [weekly(rows)], **params)
def _rows(base: dict, target: dict) -> list[list]:
return [[k, "2026-W41", v] for k, v in base.items()] + [[k, "2026-W42", v] for k, v in target.items()]
# --------------------------------------------------------------------------
# additive
# --------------------------------------------------------------------------
async def test_a_driver_is_named_with_its_share_and_the_whole_shape_is_filled():
result = await additive(label="销售额", hypothesis="华东拖累")
assert result["method"] == "decompose.additive@1" and result["status"] == "ok"
assert result["verdict"] == {"code": "driver", "subjects": [["华东"]], "direction": "down"}
assert result["phrase"] == "销售额下降 194(-8.7%),主要来自华东(-120,占 61.9%)"
assert result["values"]["delta_total"] == "-194" and result["values"]["top_key"] == "华东"
assert result["values"]["top_share"] == "0.618556701031" and result["values"]["segments"] == "4"
table = result["table"]
assert table["columns"] == ["region", "base", "target", "delta", "pct", "share", "flag"]
assert table["labels"]["region"] == "地区" and table["labels"]["share"] == "占变化的比重"
assert [r[0] for r in table["rows"]] == ["华东", "华北", "华南", "西南"] # 按变化的绝对值从大到小
assert result["settings"] == {"main_share": 0.5, "concentration": 0.8, "offset_ratio": 0.25, "flat_pct": 0.005,
"table_top_k": 10}
assert result["inputs"] == [{"ref": "Q2", "artifact": weekly()["artifact"], "role": "data",
"columns": ["region", "week", "gmv"], "rows": "all"}]
assert result["hypothesis"] == "华东拖累" and result["probes"] == []
assert result["criterion"].endswith("以各段之和为总数")
assert {c["code"] for c in result["caveats"]} == {"headline_basis"}
assert result["params"]["catalog"] == {"labels": {"region": "地区"}, "measures": {"gmv": "flow"}}
async def test_the_segments_add_up_to_the_total_digit_for_digit():
rng = random.Random(7)
regions = ["华东", "华南", "华北", "西南", "东北", "西北"]
for _ in range(20):
base = {r: round(rng.uniform(0, 5000), rng.choice([0, 2, 3])) for r in regions}
target = {r: round(rng.uniform(0, 5000), rng.choice([0, 2, 3])) for r in regions}
result = await additive(_rows(base, target))
rows = result["table"]["rows"]
assert sum(Decimal(r[3]) for r in rows) == Decimal(result["values"]["delta_total"])
assert sum(Decimal(r[1]) for r in rows) == Decimal(result["values"]["base_total"])
assert sum(Decimal(r[2]) for r in rows) == Decimal(result["values"]["target_total"])
assert Decimal(result["values"]["target_total"]) - Decimal(result["values"]["base_total"]) == \
Decimal(result["values"]["delta_total"])
assert result["values"]["gross_up"] and result["values"]["gross_down"]
async def test_concentrated_and_dispersed():
base = {f"店{i}": 1000 for i in range(10)}
target = {**base, "店0": 1450, "店1": 1400, "店2": 1050, "店3": 1050, "店4": 1050}
result = await additive(_rows(base, target), by="region")
assert result["verdict"]["code"] == "concentrated"
assert result["verdict"]["subjects"] == [["店0"], ["店1"]]
assert result["values"]["top_k"] == "2" and result["values"]["top_k_share"] == "0.85"
assert "集中在店0、店1(合计占 85.0%)" in result["phrase"]
spread = await additive(_rows({r: 1000 for r in "甲乙丙丁戊"}, {r: 1100 for r in "甲乙丙丁戊"}))
assert spread["verdict"] == {"code": "dispersed", "subjects": [], "direction": "up"}
assert "分散在各地区,没有哪一段占到一半" in spread["phrase"]
async def test_offsetting_segments_are_said_and_a_share_can_pass_one_hundred_percent():
result = await additive(_rows({"华东": 1000, "华北": 500, "华南": 500}, {"华东": 850, "华北": 540, "华南": 490}))
assert result["values"]["delta_total"] == "-120"
assert result["verdict"]["code"] == "driver" and result["values"]["top_share"] == "1.25"
offset = next(c for c in result["caveats"] if c["code"] == "offset")
assert offset["text"] == "下降的段合计 -160,上升的段合计 +40,互相抵消了一部分"
assert result["phrase"] == "销售额下降 120(-6.0%),主要来自华东(-150,占 125.0%);下降的段合计 -160,上升的段合计 +40,互相抵消了一部分"
async def test_flat_with_offsets_says_what_moved_inside():
result = await additive(_rows({"华东": 1000, "华北": 1000, "华南": 220}, {"华东": 1120, "华北": 882, "华南": 220}))
assert result["verdict"] == {"code": "flat", "subjects": [], "direction": None}
assert result["phrase"] == "销售额基本持平(+0.1%),内部此消彼长:华东 +120、华北 -118"
async def test_a_headline_that_does_not_close_stops_the_concentration_verdict():
head = snap("Q1", "SELECT week, SUM(amount) AS gmv FROM orders GROUP BY week", ["week", "gmv"],
[["2026-W42", 2050], ["2026-W41", 2220]], types={"week": "text", "gmv": "number"})
result = await additive(snaps=[weekly(), head], headline={"base": "Q1.r1.gmv", "target": "Q1.r0.gmv"})
assert result["verdict"] == {"code": "unclosed", "subjects": [], "direction": None}
assert result["values"]["gap"] == "24" and result["values"]["top_share"] is None
assert all(r[5] is None for r in result["table"]["rows"]) # 不给占比
assert result["phrase"] == "按地区拆开后各段之和和总数对不上(差 24),不判断变化来自哪里"
assert [i["role"] for i in result["inputs"]] == ["headline_base", "headline_target", "data"]
assert result["inputs"][0] == {"ref": "Q1.r1.gmv", "artifact": head["artifact"], "role": "headline_base",
"row": 1, "column": "gmv", "value": "2220"}
assert "各段之和要和总数对得上" in result["criterion"]
async def test_a_rounded_headline_closes_within_its_written_precision():
rows = [["华东", "2026-W41", "820.25"], ["华东", "2026-W42", "700.12"], ["华北", "2026-W41", "1399.9"],
["华北", "2026-W42", "1325.84"]]
for written, closes in (("2220.2", True), ("2220", True), ("2221", False)):
head = snap("Q1", "SELECT SUM(amount) AS gmv FROM orders", ["gmv_base", "gmv"], [[written, "2025.96"]])
result = await additive(rows, snaps=[weekly(rows), head], headline=["Q1.r0.gmv_base", "Q1.r0.gmv"])
assert (result["verdict"]["code"] != "unclosed") is closes, written
async def test_drilling_down_checks_closure_against_the_parent_segment():
top = await additive()
province = snap("Q4", "SELECT province, week, SUM(amount) AS gmv FROM orders WHERE region = '华东' GROUP BY 1, 2",
["province", "week", "gmv"],
[["沪", "2026-W41", 500], ["沪", "2026-W42", 410], ["苏", "2026-W41", 320], ["苏", "2026-W42", 290]])
params = {"query": "Q4", "by": ["province"], "period": "week", "base": "2026-W41", "target": "2026-W42",
"measure": "gmv", "parent": {"of": "A3", "key": ["华东"]}}
child = await analyze("decompose", "additive", snaps=[province], parents={"A3": top}, **params)
assert child["verdict"]["code"] == "driver" and child["verdict"]["subjects"] == [["沪"]]
assert child["params"]["depth"] == 1 and child["params"]["parent"] == {"of": "A3", "key": ["华东"]}
assert child["inputs"][0]["role"] == "parent" and child["inputs"][0]["ref"] == "A3"
assert "max_drill_depth" in child["settings"]
assert {c["code"] for c in child["caveats"]} == set() # 有 parent 就不是「以各段之和为总数」
short = snap("Q4", province["sql"], ["province", "week", "gmv"],
[["沪", "2026-W41", 500], ["沪", "2026-W42", 410], ["苏", "2026-W41", 300], ["苏", "2026-W42", 290]])
unclosed = await analyze("decompose", "additive", snaps=[short], parents={"A3": top}, **params)
assert unclosed["verdict"]["code"] == "unclosed" and unclosed["values"]["gap"] == "20"
# 超深:上一层已经是第 1 层,最多 1 层
error = await call_error("decompose", "additive", snaps=[province], parents={"A3": child},
settings={"max_drill_depth": 1}, **{**params, "parent": {"of": "A3", "key": ["沪"]}})
assert error.code == "too_deep" and "最多 1 层" in error.message
missing = await call_error("decompose", "additive", snaps=[province], parents={"A3": top},
**{**params, "parent": {"of": "A3", "key": ["东北"]}})
assert missing.code == "parent_key_missing"
absent = await call_error("decompose", "additive", snaps=[province], **params)
assert absent.code == "parent_missing"
async def test_new_and_vanished_segments_and_empty_cells_are_flagged():
rows = [["华东", "2026-W41", 800], ["华东", "2026-W42", 700], ["西南", "2026-W42", 90],
["东北", "2026-W41", 40], ["华北", "2026-W41", None], ["华北", "2026-W42", 30]]
result = await additive(rows)
flags = {r[0]: r[6] for r in result["table"]["rows"]}
assert flags == {"华东": None, "西南": "新出现", "东北": "消失", "华北": "为空"}
null = next(c for c in result["caveats"] if c["code"] == "null_values")
assert null["text"] == "华北的销售额为空,按 0 算"
assert next(r for r in result["table"]["rows"] if r[0] == "西南")[4] is None # 基期为 0 不给变化率
async def test_two_dimension_keys():
sql = "SELECT region, channel, week, SUM(amount) AS gmv FROM orders GROUP BY 1, 2, 3"
rows = [["华东", "线上", "W1", 500], ["华东", "线上", "W2", 300], ["华东", "线下", "W1", 300], ["华东", "线下", "W2", 310],
["华北", "线上", "W1", 200], ["华北", "线上", "W2", 205]]
s = snap("Q2", sql, ["region", "channel", "week", "gmv"], rows)
result = await analyze("decompose", "additive", snaps=[s], query="Q2", by=["region", "channel"], period="week",
base="W1", target="W2", measure="gmv")
assert result["verdict"]["subjects"] == [["华东", "线上"]]
assert "主要来自华东·线上" in result["phrase"]
async def test_ratio_and_non_additive_measures_are_call_errors():
avg = snap("Q2", "SELECT region, week, AVG(amount) AS aov FROM orders GROUP BY 1, 2", ["region", "week", "aov"], WEEKLY)
error = await call_error("decompose", "additive", snaps=[avg], query="Q2", by="region", period="week",
base="2026-W41", target="2026-W42", measure="aov")
assert error.code == "ratio_as_additive" and "改用比率拆解" in error.message
rate_sql = "SELECT region, week, SUM(refund_rate) AS r FROM orders GROUP BY 1, 2"
labeled = {"shop": {"orders": notes(columns={"refund_rate": {"measure": "ratio"}})}}
error = await call_error("decompose", "additive", snaps=[snap("Q2", rate_sql, ["region", "week", "r"], WEEKLY)],
catalogs=labeled, query="Q2", by="region", period="week", base="2026-W41",
target="2026-W42", measure="r")
assert error.code == "ratio_as_additive"
# 只是推断(没人确认)的照算,写附带说明
guessed = {"shop": {"orders": notes(columns={"refund_rate": {"measure": {"value": "ratio", "source": "llm",
"status": "proposed"}}})}}
result = await analyze("decompose", "additive", snaps=[snap("Q2", rate_sql, ["region", "week", "r"], WEEKLY)],
catalogs=guessed, query="Q2", by="region", period="week", base="2026-W41",
target="2026-W42", measure="r")
note = next(c for c in result["caveats"] if c["code"] == "measure_unlabeled")
assert note["text"] == "目录里「r」是比率只是推断,没人确认,按可以按段相加处理"
unlabeled = await additive(snaps=[weekly()], catalogs={})
assert any(c["code"] == "measure_unlabeled" and "目录没标「gmv」的度量类型" in c["text"] for c in unlabeled["caveats"])
async def test_call_errors_tell_the_agent_how_to_fix_the_call():
dup = WEEKLY + [["华东", "2026-W42", 5]]
error = await call_error("decompose", "additive", snaps=[weekly(dup)], query="Q2", by="region", period="week",
base="2026-W41", target="2026-W42", measure="gmv")
assert error.code == "duplicate_rows" and "在 SQL 里按 地区、week 汇总" in error.message
masked = await call_error("decompose", "additive", snaps=[weekly(masked=["Region"])], query="Q2", by="region",
period="week", base="2026-W41", target="2026-W42", measure="gmv")
assert masked.code == "masked_column"
missing = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="region", period="week",
base="2026-W40", target="2026-W42", measure="gmv")
assert missing.code == "period_not_found" and "2026-W41" in missing.message
column = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="area", period="week",
base="2026-W41", target="2026-W42", measure="gmv")
assert column.code == "unknown_column" and "region、week、gmv" in column.message
text = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="gmv", period="week",
base="2026-W41", target="2026-W42", measure="region")
assert text.code == "not_numeric"
unknown = await call_error("decompose", "additive", snaps=[weekly()], query="Q9", by="region", period="week",
base="2026-W41", target="2026-W42", measure="gmv")
assert unknown.code == "unknown_query"
same_column = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="week", period="week",
base="2026-W41", target="2026-W42", measure="gmv")
assert same_column.code == "bad_by"
async def test_truncated_snapshots_and_single_segments_are_refused():
result = await additive(snaps=[weekly(truncated=True)])
assert result["status"] == "refused" and result["reason_code"] == "truncated"
assert result["reason"] == "Q2 只取回了前 8 行,各段之和不是完整的和"
assert result["phrase"] == "变化拆解没有做成:Q2 只取回了前 8 行,各段之和不是完整的和"
assert result["verdict"] is None and result["table"] is None and result["values"] == {}
one = await additive([["华东", "2026-W41", 1], ["华东", "2026-W42", 2]])
assert one["reason_code"] == "too_few_segments"
async def test_two_snapshots_check_comparability_first():
base = snap("Q1", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' AND "
"order_date < '2026-10-12' GROUP BY region", ["region", "gmv"],
[["华东", 820], ["华北", 600], ["华南", 700]])
target = snap("Q2", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-12' AND "
"order_date < '2026-10-19' GROUP BY region", ["region", "gmv"],
[["华东", 700], ["华北", 560], ["华南", 666]])
probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")})
result = await analyze("decompose", "additive", snaps=[base, target], probe=probe, base_query="Q1",
target_query="Q2", by="region", measure="gmv")
assert result["verdict"]["code"] == "driver" and result["params"]["today"] == "2026-10-21"
assert not any(c["code"] == "comparability" for c in result["caveats"])
assert [i["role"] for i in result["inputs"]] == ["base", "target"]
assert result["probes"] == [{"source": "shop", "table": "orders", "column": "order_date", "min": "2025-01-01",
"max": "2026-10-20", "basis": "live", "at": "2026-10-21T09:00:00"}]
# 本期没过完:照样拆,判定后附上可比性结论
partial = snap("Q2", target["sql"].replace("'2026-10-19'", "'2026-10-25'"), ["region", "gmv"], target["rows"])
result = await analyze("decompose", "additive", snaps=[base, partial], probe=probe, base_query="Q1",
target_query="Q2", by="region", measure="gmv")
note = next(c for c in result["caveats"] if c["code"] == "comparability")
assert note["text"].startswith("两期不可比:") and "本期数据只到 2026-10-20,还差 4 天" in note["text"]
assert result["verdict"]["code"] == "driver" and note["text"] in result["phrase"]
# 口径结构不同:不拆
other = snap("Q2", "SELECT region, SUM(amount) AS gmv FROM orders JOIN refunds USING (order_id) WHERE order_date "
">= '2026-10-12' GROUP BY region", ["region", "gmv"], target["rows"])
refused = await analyze("decompose", "additive", snaps=[base, other], probe=probe, base_query="Q1",
target_query="Q2", by="region", measure="gmv")
assert refused["reason_code"] == "incomparable_sides" and refused["reason"].startswith("两份查询的口径结构不同")
# --------------------------------------------------------------------------
# ratio
# --------------------------------------------------------------------------
RATIO_SQL = "SELECT channel, week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY 1, 2"
def _ratio_snap(rows):
return snap("Q1", RATIO_SQL, ["channel", "week", "refunds", "orders"], rows)
async def ratio(rows, **params):
params = {"query": "Q1", "by": "channel", "period": "week", "base": "W41", "target": "W42",
"numerator": "refunds", "denominator": "orders", **params}
return await analyze("decompose", "ratio", snaps=[_ratio_snap(rows)], **params)
def test_mix_plus_rate_effects_equal_the_ratio_change_exactly():
rng = random.Random(11)
for _ in range(200):
segs = []
for _ in range(rng.randint(2, 7)):
d_b, d_t = rng.choice([0, rng.randint(1, 5000)]), rng.randint(1, 5000)
segs.append((Fraction(rng.randint(0, d_b)) if d_b else Fraction(0), Fraction(d_b),
Fraction(rng.randint(0, d_t)), Fraction(d_t)))
if sum(s[1] for s in segs) == 0:
continue
R, R2, effects = ratio_effects(segs)
assert sum((e[4] for e in effects), Fraction(0)) + sum((e[5] for e in effects), Fraction(0)) == R2 - R
assert sum((e[0] for e in effects), Fraction(0)) == 1 == sum((e[1] for e in effects), Fraction(0))
async def test_simpson_each_channel_improves_but_the_mix_makes_the_whole_worse():
rows = [["线上", "W41", 50, 500], ["线上", "W42", 90, 1000], ["线下", "W41", 10, 1000], ["线下", "W42", 4, 500]]
result = await ratio(rows, label="退款率")
assert result["verdict"] == {"code": "mix_driven", "subjects": [["线上"]], "direction": "up"}
assert result["phrase"].startswith("退款率上升 2.3 个百分点(4.0% 到 6.3%),主要来自结构变化:线上占比从 33.3% 到 66.7%")
simpson = next(c for c in result["caveats"] if c["code"] == "simpson")
assert simpson["text"] == "各段的退款率大多在下降,整体却上升:变化来自结构,线上占比从 33.3% 升到 66.7%"
values = result["values"]
assert Fraction(values["mix_total"]) + Fraction(values["rate_total"]) == pytest.approx(Fraction(values["delta"]))
assert result["table"]["columns"][:5] == ["channel", "n_base", "n_target", "d_base", "d_target"]
async def test_rate_driven_both_and_small_denominators():
rows = [["线上", "W41", 50, 1000], ["线上", "W42", 80, 1000], ["线下", "W41", 20, 1000], ["线下", "W42", 26, 1000]]
result = await ratio(rows, label="退款率")
assert result["verdict"]["code"] == "rate_driven" and result["verdict"]["subjects"] == [["线上"]]
assert result["phrase"] == "退款率上升 1.8 个百分点(3.5% 到 5.3%),主要来自各段自身的变化,线上影响最大"
tiny = [["线上", "W41", 2, 20], ["线上", "W42", 3, 25], ["线下", "W41", 20, 1000], ["线下", "W42", 30, 1000]]
small = await ratio(tiny, label="退款率")
note = next(c for c in small["caveats"] if c["code"] == "small_denominator")
assert note["text"] == "线上的 orders 不到 30,比率波动大,不参与多数判断"
assert "分母小" in next(r for r in small["table"]["rows"] if r[0] == "线上")[-1]
async def test_ratio_refusals():
zero = [["线上", "W41", 0, 0], ["线上", "W42", 3, 100], ["线下", "W41", 0, 0], ["线下", "W42", 2, 100]]
result = await ratio(zero)
assert result["reason_code"] == "zero_base" and result["reason"] == "基期的 orders 合计为 0,算不了比率"
orphan = [["线上", "W41", 5, 0], ["线上", "W42", 3, 100], ["线下", "W41", 1, 100], ["线下", "W42", 2, 100]]
assert (await ratio(orphan))["reason_code"] == "zero_base"
# --------------------------------------------------------------------------
# factor
# --------------------------------------------------------------------------
def _factor_snap(rows):
return snap("Q1", "SELECT week, SUM(amount) AS gmv, COUNT(*) AS orders, ROUND(AVG(amount), 2) AS aov FROM orders "
"GROUP BY week", ["week", "gmv", "orders", "aov"], rows)
async def factor(rows, **params):
params = {"query": "Q1", "period": "week", "base": "W41", "target": "W42", "product": "gmv",
"factors": ["orders", "aov"], "label": "销售额", **params}
return await analyze("decompose", "factor", snaps=[_factor_snap(rows)], **params)
def test_lmdi_contributions_add_up_to_the_change_within_float_error():
rng = random.Random(3)
for _ in range(200):
k = rng.randint(2, 4)
base = [rng.uniform(0.5, 500) for _ in range(k)]
target = [b * rng.uniform(0.5, 1.8) for b in base]
m_b = math.prod(base) * (1 + rng.uniform(-1e-6, 1e-6)) # 取整过的因子:乘积差一点点
m_t = math.prod(target) * (1 + rng.uniform(-1e-6, 1e-6))
contrib = lmdi(base, target, m_b, m_t)
assert math.fsum(contrib) == pytest.approx(m_t - m_b, rel=1e-9, abs=1e-9 * max(m_b, m_t))
same = lmdi([2.0, 3.0], [2.0, 3.0], 6.0, 6.0)
assert same == [0.0, 0.0]
async def test_factor_driver_with_rounded_factors():
rows = [["W41", 1000, 137, "7.30"], ["W42", 1320, 160, "8.25"]]
result = await factor(rows)
assert result["verdict"]["code"] in ("driver", "both")
contrib = [Fraction(r[4]) for r in result["table"]["rows"]]
assert float(sum(contrib)) == pytest.approx(320, rel=1e-9)
clean = await factor([["W41", "1000", 100, "10"], ["W42", "1320", 120, "11"]])
assert clean["verdict"] == {"code": "driver", "subjects": [["orders"]], "direction": "up"}
assert clean["phrase"].startswith("销售额上升 320(+32.0%),主要来自 orders 的变化(贡献 +")
assert clean["table"]["columns"] == ["factor", "base", "target", "pct", "contrib", "share"]
async def test_factor_call_errors_and_refusals():
# 客单价写到两位小数:容差是它的半个末位(加上订单数的),差 1.5% 不是取整造成的
error = await call_error("decompose", "factor", snaps=[_factor_snap([["W41", 1000, 100, "10.00"],
["W42", 1300, 120, "11.00"]])],
query="Q1", period="week", base="W41", target="W42", product="gmv",
factors=["orders", "aov"])
assert error.code == "not_product" and "本期差" in error.message
negative = await factor([["W41", 0, 0, 10], ["W42", 1320, 120, 11]])
assert negative["reason_code"] == "non_positive"
flat = await factor([["W41", 1000, 100, 10], ["W42", 1000, 125, 8]])
assert flat["verdict"]["code"] == "flat"
few = await call_error("decompose", "factor", snaps=[_factor_snap([["W41", 1, 1, 1], ["W42", 1, 1, 1]])],
query="Q1", period="week", base="W41", target="W42", product="gmv", factors=["orders"])
assert few.code == "bad_factors"
async def test_ratio_headline_closure_and_drilling_into_a_channel():
rows = [["线上", "W41", 50, 1000], ["线上", "W42", 80, 1000], ["线下", "W41", 20, 1000], ["线下", "W42", 26, 1000]]
head = snap("Q9", "SELECT week, SUM(refund_flag) * 1.0 / COUNT(*) AS rate FROM orders GROUP BY week",
["week", "rate"], [["W41", "0.035"], ["W42", "0.053"]])
params = {"query": "Q1", "by": "channel", "period": "week", "base": "W41", "target": "W42",
"numerator": "refunds", "denominator": "orders", "label": "退款率"}
closed = await analyze("decompose", "ratio", snaps=[_ratio_snap(rows), head],
headline={"base": "Q9.r0.rate", "target": "Q9.r1.rate"}, **params)
assert closed["verdict"]["code"] == "rate_driven"
off = snap("Q9", head["sql"], ["week", "rate"], [["W41", "0.035"], ["W42", "0.06"]])
unclosed = await analyze("decompose", "ratio", snaps=[_ratio_snap(rows), off],
headline={"base": "Q9.r0.rate", "target": "Q9.r1.rate"}, **params)
assert unclosed["verdict"]["code"] == "unclosed"
assert unclosed["phrase"].startswith("按渠道拆开后各段合起来的比率和总数对不上")
# 线上按地区下钻:各地区的分子、分母之和要等于上一层线上那一段的
region = snap("Q2", "SELECT region, week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders "
"WHERE channel = '线上' GROUP BY 1, 2", ["region", "week", "refunds", "orders"],
[["华东", "W41", 30, 600], ["华东", "W42", 55, 600], ["华北", "W41", 20, 400], ["华北", "W42", 25, 400]])
child = await analyze("decompose", "ratio", snaps=[region], parents={"A1": closed},
**{**params, "query": "Q2", "by": "region", "parent": {"of": "A1", "key": ["线上"]}})
assert child["verdict"]["code"] != "unclosed" and child["params"]["depth"] == 1
wrong = await call_error("decompose", "additive", snaps=[weekly()], parents={"A1": closed}, query="Q2",
by="region", period="week", base="2026-W41", target="2026-W42", measure="gmv",
parent={"of": "A1", "key": ["线上"]})
assert wrong.code == "parent_not_decompose"
async def test_the_same_query_cannot_be_both_periods():
error = await call_error("decompose", "additive", snaps=[weekly()], base_query="Q2", target_query="Q2",
by="region", measure="gmv")
assert error.code == "same_query"
error = await call_error("comparable", snaps=[weekly()], base_query="Q2", target_query="Q2")
assert error.code == "same_query"
+138
View File
@@ -0,0 +1,138 @@
"""差异检验(app/analysis/differ.py,细案 01 §5):比率小样本走 Fisher、均值 Welch、Holm、rows 敏感性翻转。每个用例都顺带核对
复算和运行逐字段相同。"""
from __future__ import annotations
import math
from fractions import Fraction
import pytest
from app.analysis import stats
from tests.analysis_samples import analyze, call_error, snap
RATE_SQL = "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region"
def rates(rows):
return snap("Q1", RATE_SQL, ["region", "refunds", "orders"], rows)
async def rate(rows, **params):
params = {"query": "Q1", "by": "region", "numerator": "refunds", "denominator": "orders", "label": "退款率", **params}
return await analyze("differ", "rate", snaps=[rates(rows)], **params)
async def test_small_samples_use_fisher_and_say_it_may_be_noise():
result = await rate([["西南", 2, 40], ["华东", 5, 230]])
assert result["method"] == "differ.rate@1"
assert result["verdict"] == {"code": "no_evidence", "subjects": [["西南"], ["华东"]], "direction": None}
assert result["values"]["test"] == "fisher" and result["values"]["n_a"] == "40" and result["values"]["n_b"] == "230"
assert Fraction(result["values"]["p"]) == pytest.approx(stats.fisher_exact(2, 38, 5, 225), rel=1e-9)
assert result["phrase"] == ("西南和华东的退款率差异可能只是样本波动(样本量 40、230);样本少(有格子的期望频数不到 5),"
"用了精确检验")
assert "p 值" not in result["phrase"] and "显著" not in result["phrase"]
assert result["settings"] == {"differ_alpha": 0.05, "min_expected": 5}
async def test_large_samples_use_the_pooled_z_test():
result = await rate([["西南", 80, 1000], ["华东", 40, 1000]])
assert result["verdict"] == {"code": "differs", "subjects": [["西南"], ["华东"]], "direction": "up"}
z, p = stats.pooled_z(80, 1000, 40, 1000)
assert Fraction(result["values"]["p"]) == pytest.approx(p, rel=1e-9) and result["values"]["test"] == "z"
low, high = stats.newcombe(80, 1000, 40, 1000)
assert Fraction(result["values"]["ci_lo"]) == pytest.approx(low, rel=1e-9)
assert result["values"]["a"] == "0.08" and result["values"]["diff"] == "0.04"
assert result["phrase"] == "西南的退款率(8.0%)高于华东(4.0%),差异站得住"
lower = await rate([["西南", 80, 1000], ["华东", 40, 1000]], pairs=[["华东", "西南"]])
assert lower["verdict"]["direction"] == "down" and lower["phrase"].startswith("华东的退款率(4.0%)低于西南(8.0%)")
async def test_several_groups_each_against_the_rest_with_holm():
rows = [["西南", 80, 1000], ["华东", 40, 1000], ["华南", 42, 1000], ["华北", 38, 1000]]
result = await rate(rows)
table = result["table"]
assert [r[0] for r in table["rows"]] == ["西南", "华东", "华南", "华北"]
assert table["rows"][0][1] == "其余各地区"
raw = [Fraction(r[10]) for r in table["rows"]]
adjusted = [Fraction(r[11]) for r in table["rows"]]
assert all(a >= p for a, p in zip(adjusted, raw))
assert adjusted == pytest.approx(stats.holm([float(p) for p in raw]), rel=1e-9)
assert result["verdict"]["code"] == "differs" and result["verdict"]["subjects"] == [["西南"], ["其余各地区"]]
assert any(c["code"] == "multiple_comparisons" and c["text"] == "比了 4 对,已按次数校正" for c in result["caveats"])
assert result["phrase"] == "比了 4 对,1 对差异站得住:西南的退款率(8.0%)高于其余各地区(4.0%)"
ref = await rate(rows, reference="华东")
assert [r[0] for r in ref["table"]["rows"]] == ["西南", "华南", "华北"] and ref["params"]["reference"] == "华东"
async def test_rate_call_errors():
assert (await call_error("differ", "rate", snaps=[rates([["西南", 5, 4], ["华东", 1, 9]])], query="Q1", by="region",
numerator="refunds", denominator="orders")).code == "not_count"
assert (await call_error("differ", "rate", snaps=[rates([["西南", 1.5, 4], ["华东", 1, 9]])], query="Q1",
by="region", numerator="refunds", denominator="orders")).code == "not_count"
unknown = await call_error("differ", "rate", snaps=[rates([["西南", 1, 4], ["华东", 1, 9]])], query="Q1",
by="region", numerator="refunds", denominator="orders", pairs=[["西南", "东北"]])
assert unknown.code == "unknown_group" and "西南、华东" in unknown.message
dup = await call_error("differ", "rate", snaps=[rates([["西南", 1, 4], ["西南", 1, 9]])], query="Q1", by="region",
numerator="refunds", denominator="orders")
assert dup.code == "duplicate_rows"
async def test_means_from_sums_and_squares_use_welch():
sql = "SELECT channel, SUM(amount) AS s, SUM(amount * amount) AS ss, COUNT(*) AS n FROM orders GROUP BY channel"
rows = [["线上", 5000, 300000, 100], ["线下", 4000, 200000, 100]]
result = await analyze("differ", "mean", snaps=[snap("Q1", sql, ["channel", "s", "ss", "n"], rows)], query="Q1",
by="channel", sum="s", sumsq="ss", n="n", label="客单价")
va = (300000 - 5000 ** 2 / 100) / 99
vb = (200000 - 4000 ** 2 / 100) / 99
expected = stats.welch(50, va, 100, 40, vb, 100)
assert Fraction(result["values"]["p"]) == pytest.approx(expected.p, rel=1e-9)
assert result["verdict"]["code"] == "differs" and result["values"]["test"] == "welch"
assert result["phrase"] == "线上的客单价(50)高于线下(40),差异站得住"
assert result["method"] == "differ.welch@1" and result["settings"] == {"differ_alpha": 0.05, "min_n": 30}
async def test_means_from_mean_sd_and_n_and_small_samples():
sql = "SELECT channel, avg_amount, sd_amount, n FROM channel_stats"
rows = [["线上", 52, 12, 12], ["线下", 50, 11, 400]]
result = await analyze("differ", "mean", snaps=[snap("Q1", sql, ["channel", "avg_amount", "sd_amount", "n"], rows)],
query="Q1", by="channel", mean="avg_amount", sd="sd_amount", n="n", label="客单价")
assert result["verdict"]["code"] == "no_evidence"
note = next(c for c in result["caveats"] if c["code"] == "small_sample")
assert note["text"] == "有一组的样本数不到 30(线上 12),结论不稳"
assert result["phrase"].startswith("线上和线下的客单价差异可能只是样本波动(样本量 12、400)")
async def test_rows_mode_flags_a_single_value_that_flips_the_verdict():
base = [["线下", v] for v in [40, 60] * 200]
online = [["线上", f"{v:.1f}"] for v in [41.3, 61.3] * 200]
online[-1] = ["线上", "231.3"]
s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], online + base)
result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount", label="客单价")
assert result["method"] == "differ.welch_rows@1"
assert result["verdict"]["code"] == "differs"
note = next(c for c in result["caveats"] if c["code"] == "outlier_sensitive")
assert note["text"] == "去掉最大的一笔后,差异不再站得住"
assert note["text"] in result["phrase"]
hidden = [["线上", 50 + (i % 7)] for i in range(30)] + [["线下", 48 + (i % 5)] for i in range(30)] + [["线上", 400]]
s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], hidden)
flipped = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount", label="客单价")
assert flipped["verdict"]["code"] == "no_evidence"
assert any(c["text"] == "去掉最大的一笔后,差异变得站得住" for c in flipped["caveats"])
async def test_two_periods_of_the_same_group():
sql = "SELECT week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders WHERE region = '西南' GROUP BY week"
s = snap("Q1", sql, ["week", "refunds", "orders"], [["W41", 30, 1000], ["W42", 60, 1000], ["W40", 31, 1000]])
result = await analyze("differ", "rate", snaps=[s], query="Q1", period="week", base="W41", target="W42",
numerator="refunds", denominator="orders", label="退款率")
assert result["verdict"] == {"code": "differs", "subjects": [["W42"], ["W41"]], "direction": "up"}
assert result["inputs"][0]["rows"] == [0, 1]
assert result["phrase"] == "W42 的退款率(6.0%)高于 W41(3.0%),差异站得住"
async def test_a_group_with_one_value_cannot_be_tested():
s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], [["线上", 5], ["线下", 4], ["线下", 6]])
result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount")
assert result["verdict"]["code"] == "no_evidence" and result["values"]["p"] is None
assert result["table"]["rows"][0][-1] == "样本不够,算不了"
assert math.isfinite(float(result["values"]["diff"]))
+57
View File
@@ -0,0 +1,57 @@
"""期次(app/analysis/periods.py):日期、YYYY-MM、YYYY-Www、YYYY-Qn、整数序号认得出、能排序、知道覆盖哪几天;粒度按值推断;
写法混着、带时刻的认不出。"""
from __future__ import annotations
from datetime import date
from app.analysis import periods as pd
def test_dates_infer_their_grain():
items, grain = pd.parse(["2026-10-01", "2026-10-02", "2026-10-04"])
assert grain == "day" and [i.start for i in items] == [date(2026, 10, 1), date(2026, 10, 2), date(2026, 10, 4)]
items, grain = pd.parse(["2026-09-28", "2026-10-05", "2026-10-19"]) # 隔 7 天的整数倍:周(缺一周)
assert grain == "week" and [i.ordinal for i in items] == [0, 1, 3] and items[0].end == date(2026, 10, 4)
items, grain = pd.parse(["2026-08-01", "2026-09-01", "2026-10-01"])
assert grain == "month" and items[1].end == date(2026, 9, 30)
items, grain = pd.parse(["2026-01-01", "2026-04-01", "2026-07-01"])
assert grain == "quarter" and items[0].end == date(2026, 3, 31)
items, grain = pd.parse([20261001, 20261002])
assert grain == "day" and items[1].start == date(2026, 10, 2)
assert pd.parse(["2026-10-01 00:00:00", "2026-10-02 00:00:00"])[1] == "day"
def test_labels_for_months_weeks_quarters_and_sequences():
items, grain = pd.parse(["2026-09", "2026-10"])
assert grain == "month" and items[1].ordinal - items[0].ordinal == 1 and items[1].end == date(2026, 10, 31)
items, grain = pd.parse([202609, 202610])
assert grain == "month"
items, grain = pd.parse(["2026-W41", "2026-W42", "2026-W01"])
assert grain == "week" and items[1].start == date(2026, 10, 12) and items[1].end == date(2026, 10, 18)
assert items[1].ordinal - items[0].ordinal == 1
items, grain = pd.parse(["2025-W52", "2026-W01"]) # 跨年也连着
assert items[1].ordinal - items[0].ordinal == 1
items, grain = pd.parse(["2026Q3", "2026-Q4"])
assert grain == "quarter" and items[1].start == date(2026, 10, 1)
items, grain = pd.parse([3, 1, 2])
assert grain == "seq" and items[0].start is None and [i.ordinal for i in items] == [3, 1, 2]
def test_unreadable_periods():
assert pd.parse(["2026-10-01", "2026-W41"]) is None # 写法混着
assert pd.parse(["2026-10-01 08:30:00", "2026-10-02 08:30:00"]) is None # 带时刻
assert pd.parse(["上周", "本周"]) is None
assert pd.parse(["2026-10", "2026-11"], "week") is None # 粒度和写法对不上
assert pd.parse(["2026-10-01", "2026-10-03"], "week") is None # 按周但不隔 7 天
assert pd.parse([]) is None and pd.parse([None, "2026-10-01"]) is None
def test_calendar_units_and_weekend_days():
assert pd.calendar_unit(date(2026, 9, 1), date(2026, 9, 30)) == "month"
assert pd.calendar_unit(date(2026, 7, 1), date(2026, 9, 30)) == "quarter"
assert pd.calendar_unit(date(2026, 1, 1), date(2026, 12, 31)) == "year"
assert pd.calendar_unit(date(2026, 9, 2), date(2026, 9, 30)) is None
assert pd.weekend_days(date(2026, 10, 12), date(2026, 10, 18)) == 2 # 周一到周日
assert pd.weekend_days(date(2026, 10, 16), date(2026, 10, 18)) == 2 # 周五到周日
assert pd.weekend_days(date(2026, 10, 12), date(2026, 10, 14)) == 0
assert pd.weekend_days(date(2026, 10, 1), date(2026, 10, 31)) == 9
+121
View File
@@ -0,0 +1,121 @@
"""判定语、数的写法、label 消毒、判定标准(app/analysis/phrases.py、numbers.py):一种说法,只在这里定义。"""
from __future__ import annotations
from fractions import Fraction
from app.analysis import phrases as ph
from app.analysis.contract import DEFAULTS
from app.analysis.numbers import canon, fixed, round_sig, to_fraction
def test_canonical_decimal_strings_are_plain_and_stable():
assert canon(Fraction(-120)) == "-120"
assert canon("2220.00") == "2220" # 去掉多余的 0
assert canon(0.1 + 0.2) == "0.3" # 浮点留 12 位有效数字
assert canon(Fraction(1, 3)) == "0.333333333333"
assert canon(Fraction(7, 16)) == "0.4375" # 除得尽的照写
assert canon(-0.0) == "0" and canon(Fraction(0)) == "0"
assert canon(1e-30) == "0.000000000000000000000000000001" # 不带指数
assert canon(1e20) == "100000000000000000000"
assert canon("123456789012345678.25") == "123456789012345678.25" # 快照里的数和它们的和差照原样
assert canon(None) is None and canon("abc") is None and canon(True) is None
assert to_fraction(0.1) == Fraction(1, 10) and to_fraction("45678.50") == Fraction(4567850, 100)
assert round_sig(Fraction(2, 3), 3) == Fraction(667, 1000)
def test_fixed_rounds_half_up_like_render_cell():
assert fixed(Fraction(2675, 1000), 2) == "2.68" # 2.675 不走浮点的 round(会得 2.67)
assert fixed(Fraction(-5, 100), 1, strip=False) == "-0.1"
assert fixed(Fraction(-4, 100), 1, strip=False) == "0.0" # 不写 -0.0
assert fixed(Fraction(123456789, 100), 2, group=True) == "1,234,567.89"
def test_number_formats_in_phrases():
assert ph.pct(Fraction(-87, 1000)) == "-8.7%" and ph.pct(Fraction(3, 100)) == "+3.0%"
assert ph.pct(0) == "0.0%" and ph.pct(Fraction(6186, 10000), signed=False) == "61.9%"
assert ph.pct(Fraction(4, 100000)) == "+0.004%" # 不是 0 的不写成 0.0%
assert ph.amount(Fraction(1234567891, 1000)) == "1,234,567.89"
assert ph.amount(Fraction(120)) == "120" and ph.amount(Fraction(-120), signed=True) == "-120"
assert ph.amount(Fraction(40), signed=True) == "+40"
assert ph.amount(Fraction(21, 1000)) == "0.021" and ph.amount(Fraction(4567, 10 ** 7)) == "0.00046"
assert ph.change(Fraction(-194), Fraction(2220)) == "下降 194(-8.7%)"
assert ph.change(Fraction(120), Fraction(0)) == "上升 120" # 基期为 0 不给百分比
assert ph.change(0, 5) == "没有变化"
assert ph.ratio_change(Fraction(21, 1000), Fraction(25, 1000), proportion=True) == "上升 0.4 个百分点(2.1% 到 2.5%)"
assert ph.ratio_change(Fraction(82), Fraction(852, 10), proportion=False) == "上升 3.2(+3.9%)"
assert ph.threshold_pct(0.005) == "0.5%" and ph.share_words(0.5) == "一半" and ph.share_words(0.6) == "60%"
def test_labels_are_scrubbed_of_numbers_units_and_markers():
assert ph.clean_label("销售额(同比 +30%)") == "销售额(同比)"
assert ph.clean_label("近三成客户的退款率") == "近客户的退款率"
assert ph.clean_label("销售额[[v:Q1.r0.gmv]]") == "销售额"
assert ph.clean_label("¥ 销售额 3.5万") == "销售额"
assert ph.clean_label("gmv_2026") == "gmv_"
assert len(ph.clean_label("销售额" * 30)) == ph.LABEL_MAX
assert ph.clean_label("30%") == "" and ph.clean_label(None) == ""
# 消毒后为空就退回下一个来源(目录里的叫法、列名)
assert ph.pick_label("12.5%", None, "gmv") == "gmv"
assert ph.pick_label("", "实收金额", "gmv") == "实收金额"
assert ph.pick_label("2026", None, "2026") == "这一列"
def test_numbers_stuffed_into_a_label_never_reach_the_phrase():
label = ph.pick_label("销售额(同比 +30%,3.5万)", "销售额", "gmv")
phrase = ph.decompose_driver(label, ph.change(-194, 2220), "华东", "-120", Fraction(6186, 10000))
assert "30" not in phrase and "3.5" not in phrase and "万" not in phrase
assert phrase == "销售额(同比)下降 194(-8.7%),主要来自华东(-120,占 61.9%)"
def test_spacing_between_chinese_and_ascii():
assert ph.say("gmv", "下降 194") == "gmv 下降 194"
assert ph.say("主要来自", "online", "(", "-120") == "主要来自 online(-120"
assert ph.say("销售额", "下降") == "销售额下降"
assert ph.key_text(["华东", "线上"]) == "华东·线上" and ph.key_text([None]) == "(空)"
assert ph.names(["华东", "华北", "华南"], 2) == "华东、华北等 3 段"
def test_phrase_templates():
assert ph.decompose_concentrated("销售额", "下降 194(-8.7%)", "华东、华北", Fraction(85, 100)) == \
"销售额下降 194(-8.7%),集中在华东、华北(合计占 85.0%)"
assert ph.decompose_dispersed("销售额", "下降 194(-8.7%)", "地区", 0.5) == \
"销售额下降 194(-8.7%),分散在各地区,没有哪一段占到一半"
assert ph.decompose_dispersed("销售额", "下降 194(-8.7%)", "地区", 0.6).endswith("没有哪一段占到 60%")
assert ph.decompose_flat("销售额", Fraction(2, 1000), ("华东", "+120", "华北", "-118")) == \
"销售额基本持平(+0.2%),内部此消彼长:华东 +120、华北 -118"
assert ph.decompose_unclosed("地区", Fraction(-12), ratio=False) == "按地区拆开后各段之和和总数对不上(差 12),不判断变化来自哪里"
assert ph.baseline_phrase("outside_band", "销售额", "本期变化 -8.7%", 12, "周", "-6.1% 至 +5.3%") == \
"销售额本期变化 -8.7%,超出过去 12 周的常规波动范围(-6.1% 至 +5.3%)"
assert ph.trend_phrase("rising", "线上销售额", 10, "周", "周", "+3.0%", (4, "up")) == \
"过去 10 周线上销售额呈上升趋势(每周约 +3.0%),已连续 4 周上升"
assert ph.trend_phrase("no_trend", "销售额", 8, "个月", "月", None, None) == "过去 8 个月销售额没有明显的上升或下降趋势"
assert ph.differ_phrase([ph.differ_pair("西南", "华东", "退款率", "5.0%", "2.1%", True)], 1) == \
"西南的退款率(5.0%)高于华东(2.1%),差异站得住"
assert ph.no_evidence_phrase("西南", "华东", "退款率", ("40", "230"), 1) == "西南和华东的退款率差异可能只是样本波动(样本量 40、230)"
assert ph.comparable_phrase("incomparable", ["本期数据只到 2026-10-15,还差 3 天"], []) == \
"两期不可比:本期数据只到 2026-10-15,还差 3 天"
assert ph.comparable_phrase("comparable", [], ["整期"]) == "两期可比,整期核对不了"
assert ph.refused("baseline", "只有 5 期历史,至少要 8 期") == "波动基线没有做成:只有 5 期历史,至少要 8 期"
assert ph.current_text(Fraction(-31, 1000), basis="change", lag=7, grain="day") == "比上周同一天变化 -3.1%"
assert ph.current_text(Fraction(1, 10), basis="change", lag=52, grain="week") == "同比变化 +10.0%"
def test_caveats_follow_the_phrase_at_most_two_in_order():
caveats = [ph.caveat("measure_unlabeled", "目录没标"), ph.caveat("null_values", "西南为空"),
ph.caveat("offset", "互相抵消"), ph.caveat("comparability", "两期有条件可比:周末不同")]
assert ph.with_caveats("判定", caveats) == "判定;两期有条件可比:周末不同;互相抵消"
assert ph.with_caveats("判定", caveats, skip=("offset",)) == "判定;两期有条件可比:周末不同;西南为空"
assert ph.with_caveats("判定", [ph.caveat("headline_basis", "以各段之和为总数")]) == "判定"
def test_criteria_are_written_with_the_actual_thresholds():
text = ph.criterion_additive(DEFAULTS, top_k_max=2, summed=True)
assert text == ("一段占总变化的一半以上算主要来源,最多前 2 段合计占 80% 以上算集中,总变化不到基期的 0.5% 算基本持平,"
"以各段之和为总数")
custom = {**DEFAULTS, "main_share": 0.6, "flat_pct": 0.01}
assert "占总变化的 60% 以上" in ph.criterion_additive(custom, top_k_max=3, summed=False)
assert "1% 算基本持平" in ph.criterion_additive(custom, top_k_max=3, summed=False)
assert "差不到 2 倍稳健标准差" in ph.criterion_baseline(DEFAULTS, 12, "周")
assert "3 倍以上算明显超出" in ph.criterion_baseline(DEFAULTS, 12, "周")
assert "小于 0.05" in ph.criterion_trend(DEFAULTS, 10, "周")
assert "Fisher" in ph.criterion_differ(DEFAULTS, "rate") and "Welch" in ph.criterion_differ(DEFAULTS, "mean")
+129
View File
@@ -0,0 +1,129 @@
"""可比性检查的现查(app/analysis/probe.py):make_prober 在真的数据源上探表的日期范围、在库里算按运行当天现算的界,遮罩列
不探、超时限不探、现查不成退回剖析事实;Book 先登记、查完再算,结论只记最后真用到的。库是现造的 SQLite 文件,表名、列名、
数字都是虚构的。"""
from __future__ import annotations
import sqlite3
import uuid
from datetime import date, datetime, timedelta, timezone
import pytest
from app import analysis
from app.analysis.contract import effective_settings, same
from app.analysis.probe import Book, make_prober
from app.data import introspect
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource
from tests.analysis_samples import CATALOGS, recomputed, snap
CLOCK = "2026-10-21T09:30:00"
#: SQLite 的 date('now') 按 UTC 算:表里的数据造到 UTC 的昨天
UTC_TODAY = datetime.now(timezone.utc).date()
LAST_DAY = UTC_TODAY - timedelta(days=1)
@pytest.fixture
async def shop(tmp_path):
"""orders:order_date 从 2026-01-01 到昨天(UTC),每天一单;customers 的 signup_date、phone 是遮罩列。"""
path = tmp_path / f"{uuid.uuid4().hex[:8]}.db"
db = sqlite3.connect(path)
db.executescript("CREATE TABLE orders (order_date TEXT, region TEXT, amount INTEGER);"
"CREATE TABLE customers (signup_date TEXT, phone TEXT);"
"INSERT INTO customers VALUES ('2026-01-05', '13900000000');")
day = date(2026, 1, 1)
while day <= LAST_DAY:
db.execute("INSERT INTO orders VALUES (?, '华东', 100)", (day.isoformat(),))
day += timedelta(days=1)
db.commit()
db.close()
name = f"shop_{uuid.uuid4().hex[:8]}"
async with SessionLocal() as session:
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True,
options={"mask_columns": ["phone", "signup_date"]})
session.add(row)
await session.flush()
row.schema_cache = await introspect.introspect(row)
await session.commit()
await session.refresh(row)
yield row
await engines.invalidate(row.id)
async def test_the_live_range_of_a_column(shop):
prober = make_prober({shop.name: shop}, clock=lambda: CLOCK)
probe = await prober(shop.name, "orders", "order_date")
assert probe == {"source": shop.name, "table": "orders", "column": "order_date", "min": "2026-01-01",
"max": LAST_DAY.isoformat(), "basis": "live", "at": CLOCK}
assert prober.dialect(shop.name) == "sqlite" and prober.dialect("nope") is None
assert await prober("nope", "orders", "order_date") is None
async def test_masked_columns_are_never_probed(shop):
prober = make_prober({shop.name: shop})
assert await prober(shop.name, "customers", "signup_date") is None # 数据源设的遮罩列
explicit = make_prober({shop.name: shop}, masked={shop.name: ["ORDER_DATE"]})
assert await explicit(shop.name, "orders", "order_date") is None
async def test_a_failed_probe_falls_back_to_profile_facts(shop):
async def facts(table):
return {"at": "2026-10-20T02:00:00", "date_ranges": [{"column": "paid_on", "min": "2026-02-01",
"max": "2026-10-19"}]}
prober = make_prober({shop.name: shop}, facts={shop.name: facts}, clock=lambda: CLOCK)
probe = await prober(shop.name, "payments", "paid_on") # 没有这张表:现查不成
assert probe == {"source": shop.name, "table": "payments", "column": "paid_on", "min": "2026-02-01",
"max": "2026-10-19", "basis": "profile", "at": "2026-10-20"}
async def test_the_budget_stops_further_probes(shop):
prober = make_prober({shop.name: shop}, budget=0)
await prober(shop.name, "orders", "order_date")
assert await prober(shop.name, "orders", "order_date") is None
async def test_bounds_are_computed_in_the_database(shop):
prober = make_prober({shop.name: shop}, clock=lambda: CLOCK)
probe = await prober.bounds(shop.name, "DATE('now', '-7 days')")
assert probe["expr"] == "DATE('now', '-7 days')" and probe["basis"] == "live" and probe["at"] == CLOCK
assert date.fromisoformat(probe["value"]) == UTC_TODAY - timedelta(days=7)
assert await prober.bounds(shop.name, "no_such_function(1)") is None
def test_the_book_registers_first_and_records_only_what_was_used():
book = Book({})
assert book.range("shop", "orders", "order_date") is None and book.bound("shop", "DATE('now')") is None
assert book.missing == [("range", "shop", "orders", "order_date"), ("bound", "shop", "DATE('now')")]
found = {"source": "shop", "table": "orders", "column": "order_date", "min": "2026-01-01", "max": "2026-10-18",
"basis": "live", "at": CLOCK}
filled = Book({("range", "shop", "orders", "order_date"): found, ("bound", "shop", "DATE('now')"): None})
assert filled.range("shop", "ORDERS", "Order_Date") is found # 表名、列名不分大小写
filled.bound("shop", "DATE('now')")
assert filled.missing == [] and filled.used == [found]
replay = Book([found]) # 复算:记下的就是全部
assert replay.bound("shop", "DATE('now', '-1 day')") is None and replay.missing == []
async def test_a_full_comparability_check_against_the_database_records_its_probes(shop):
sql = ("SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', '{a}') "
"AND order_date < date('now', '{b}')")
base = snap("Q1", sql.format(a="-14 days", b="-7 days"), ["gmv"], [[700]], source=shop.name)
target = snap("Q2", sql.format(a="-7 days", b="0 days"), ["gmv"], [[300]], source=shop.name)
prober = make_prober({shop.name: shop}, clock=lambda: CLOCK)
result = await analysis.run({"op": "comparable", "params": {"base_query": "Q1", "target_query": "Q2",
"measure": "gmv"}},
{"Q1": base, "Q2": target}.get, settings=effective_settings(None),
context={"today": UTC_TODAY, "probe": prober,
"catalogs": {shop.name: CATALOGS["shop"]}})
exprs = [p["expr"] for p in result["probes"] if "expr" in p]
assert exprs == ["DATE('now', '-14 days')", "DATE('now', '-7 days')", "DATE('now', '0 days')"]
ranges = [p for p in result["probes"] if "table" in p]
assert [(p["table"], p["column"], p["max"], p["basis"]) for p in ranges] == \
[("orders", "order_date", LAST_DAY.isoformat(), "live")]
assert result["values"]["base_days"] == "7" and result["values"]["target_days"] == "7"
assert result["values"]["target_end"] == LAST_DAY.isoformat()
assert all(i["dialect"] == "sqlite" for i in result["inputs"])
assert result["verdict"]["code"] == "comparable" and result["phrase"] == "两期可比"
assert same(result, recomputed(result, [base, target]))
+152
View File
@@ -0,0 +1,152 @@
"""analysis.run / recompute(app/analysis/__init__.py):结论的形状严格按契约、同输入两次 run 逐字节相同、复算和运行逐字段相同且
只用记下的 probes(不现查、不读目录)、调用方写的系统字段不收、复算取不回输入时的说法。"""
from __future__ import annotations
import json
from datetime import date, timedelta
import pytest
from app import analysis
from app.analysis import contract as c
from app.analysis.contract import CallError, effective_settings, same
from tests.analysis_samples import CATALOGS, analyze, call_error, fixed_prober, recomputed, snap
WEEKLY_SQL = "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week"
WEEKLY = snap("Q2", WEEKLY_SQL, ["region", "week", "gmv"],
[["华东", "2026-W41", 820], ["华东", "2026-W42", 700], ["华北", "2026-W41", 600],
["华北", "2026-W42", 560], ["华南", "2026-W41", 700], ["华南", "2026-W42", 666]])
SERIES = snap("Q1", "SELECT DATE(order_date, 'weekday 0', '-6 days') AS week, SUM(amount) AS gmv FROM orders GROUP BY 1",
["week", "gmv"], [[(date(2026, 6, 1) + timedelta(days=7 * i)).isoformat(), 100 + (i % 4) * 3 + i]
for i in range(20)])
RATES = snap("Q3", "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region",
["region", "refunds", "orders"], [["西南", 80, 1000], ["华东", 40, 1000], ["华北", 41, 1000]])
BASE = snap("Q4", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' AND "
"order_date < '2026-10-12' GROUP BY region", ["region", "gmv"], [["华东", 820], ["华北", 600]])
TARGET = snap("Q5", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-12' AND "
"order_date < '2026-10-19' GROUP BY region", ["region", "gmv"], [["华东", 700], ["华北", 590]])
SNAPS = [WEEKLY, SERIES, RATES, BASE, TARGET]
PROBE = {("orders", "order_date"): ("2025-01-01", "2026-10-20")}
CALLS = [
("decompose", "additive", {"query": "Q2", "by": "region", "period": "week", "base": "2026-W41",
"target": "2026-W42", "measure": "gmv", "label": "销售额"}),
("decompose", "additive", {"base_query": "Q4", "target_query": "Q5", "by": "region", "measure": "gmv"}),
("baseline", None, {"query": "Q1", "period": "week", "measure": "gmv"}),
("trend", None, {"query": "Q1", "period": "week", "measure": "gmv"}),
("differ", "rate", {"query": "Q3", "by": "region", "numerator": "refunds", "denominator": "orders"}),
("comparable", None, {"base_query": "Q4", "target_query": "Q5", "measure": "gmv"}),
]
async def _run(op, mode, params, *, probe=None, **kw):
return await analyze(op, mode, snaps=SNAPS, probe=probe or fixed_prober(PROBE), check=False, **params, **kw)
def _assert_shape(result):
"""结论的字段、取值严格按契约。"""
assert set(result) == set(c.AnalysisResult.__annotations__)
assert result["op"] in c.OPS and result["method"] == c.METHODS[(result["op"], result["mode"])]
assert result["status"] in ("ok", "refused")
if result["status"] == "refused":
assert result["reason_code"] in c.REASONS and result["reason"] and result["verdict"] is None
else:
assert result["reason_code"] is None and result["reason"] is None
assert result["verdict"]["code"] in c.VERDICTS[result["op"]]
assert set(result["verdict"]) == {"code", "subjects", "direction"}
assert all(isinstance(s, list) and all(isinstance(x, str) for x in s) for s in result["verdict"]["subjects"])
assert result["verdict"]["direction"] in (None, "up", "down")
table = result["table"]
assert set(table) == {"columns", "labels", "rows"} and set(table["labels"]) == set(table["columns"])
assert all(len(r) == len(table["columns"]) for r in table["rows"])
assert all(v is None or isinstance(v, str) for v in result["values"].values())
assert all(set(cv) == {"code", "text"} and cv["code"] in c.CAVEATS for cv in result["caveats"])
assert set(result["settings"]) <= set(c.DEFAULTS)
assert all(i.get("artifact") for i in result["inputs"])
assert result["criterion"] and result["phrase"]
json.dumps(result, ensure_ascii=False) # 写得进 JSON
@pytest.mark.parametrize("op,mode,params", CALLS)
async def test_every_result_has_the_contract_shape_and_recomputes_the_same(op, mode, params):
result = await _run(op, mode, params)
_assert_shape(result)
again = recomputed(result, SNAPS)
assert same(result, again)
assert json.dumps(again, sort_keys=True, ensure_ascii=False) == json.dumps(result, sort_keys=True, ensure_ascii=False)
@pytest.mark.parametrize("op,mode,params", CALLS)
async def test_the_same_inputs_give_byte_identical_results(op, mode, params):
one = await _run(op, mode, params)
two = await _run(op, mode, params)
assert json.dumps(one, sort_keys=True, ensure_ascii=False) == json.dumps(two, sort_keys=True, ensure_ascii=False)
async def test_recompute_uses_the_recorded_probes_and_facts_not_live_ones():
result = await _run("comparable", None, {"base_query": "Q4", "target_query": "Q5", "measure": "gmv"})
assert result["verdict"]["code"] == "comparable" and len(result["probes"]) == 1
# 记下的范围换一个(表里数据只到 10-16):复算按记下的算,结论跟着变——它没有现查
edited = {**result, "probes": [{**result["probes"][0], "max": "2026-10-16"}]}
assert recomputed(edited, SNAPS)["verdict"]["code"] == "incomparable"
# 目录事后改了也不影响复算:目录事实记在 params 里
decompose = await _run(*CALLS[0])
assert decompose["params"]["catalog"]["labels"]["region"] == "地区"
CATALOGS["shop"]["orders"]["columns"]["region"]["label"]["value"] = "大区"
try:
assert same(decompose, recomputed(decompose, SNAPS))
finally:
CATALOGS["shop"]["orders"]["columns"]["region"]["label"]["value"] = "地区"
async def test_system_params_written_by_the_caller_are_ignored():
params = {"query": "Q1", "period": "week", "measure": "gmv", "today": "2020-01-01",
"catalog": {"labels": {"gmv": "假的叫法"}}, "depth": 9}
result = await analysis.run({"op": "baseline", "params": params}, {s["ref"]: s for s in SNAPS}.get,
settings=effective_settings(None),
context={"today": date(2026, 10, 21), "catalogs": CATALOGS})
assert result["params"]["today"] == "2026-10-21"
assert "假的叫法" not in result["phrase"] and "depth" not in result["params"]
async def test_analysis_card_overrides_reach_the_criterion_and_settings():
params = CALLS[0][2]
result = await _run("decompose", "additive", params, settings={"main_share": 0.6})
assert result["settings"]["main_share"] == 0.6 and "占总变化的 60% 以上" in result["criterion"]
async def test_call_errors_for_unknown_ops_and_modes():
with pytest.raises(CallError) as e:
await analysis.run({"op": "forecast", "params": {}}, {}.get, settings=effective_settings(None), context={})
assert e.value.code == "unknown_op" and "变化拆解" in e.value.message
with pytest.raises(CallError) as e:
await analysis.run({"op": "decompose", "mode": "share", "params": {}}, {}.get,
settings=effective_settings(None), context={})
assert e.value.code == "unknown_mode" and "additive / ratio / factor" in e.value.message
missing = await call_error("baseline", snaps=SNAPS, query="Q1", period="week")
assert missing.code == "missing_param" and missing.message == "没写度量列"
async def test_recompute_says_when_an_input_cannot_be_loaded():
result = await _run(*CALLS[0])
with pytest.raises(CallError) as e:
analysis.recompute(result, lambda artifact: None)
assert e.value.code == "input_missing" and e.value.message == "复算取不回 Q2 的查询快照"
async def test_the_hypothesis_is_recorded_but_never_rendered():
result = await _run(*CALLS[0], hypothesis="可能是华东促销结束 30% 的影响")
assert result["hypothesis"] == "可能是华东促销结束 30% 的影响"
assert "促销" not in result["phrase"] and "30" not in result["phrase"]
assert same(result, {**recomputed(result, SNAPS), "hypothesis": None})
async def test_ledger_entries_name_the_real_input_snapshots():
result = await _run(*CALLS[1])
entry = c.ledger_entry(result, node_id="agent", exec_no=1, artifact="f" * 64, call_id="call-1")
assert entry["inputs"] == [BASE["artifact"], TARGET["artifact"]] and entry["op"] == "decompose"
def test_the_call_limit_error_is_spoken_with_the_limit():
error = analysis.exhausted({"max_calls": 5})
assert error.code == "max_calls" and error.message == "这次运行的分析次数已经用完(最多 5 次):用已有的分析结论写,不再新做分析"
+210
View File
@@ -0,0 +1,210 @@
"""波动基线、趋势(app/analysis/series.py,细案 01 §3、§4):常规 / 偏离 / 明显偏离、σ̂ = 0、历史不够、本期不完整、缺期、日数据
lag 7、新高新低;上升 / 下降 / 无趋势、连续期数、缺期。每个用例都顺带核对复算和运行逐字段相同。"""
from __future__ import annotations
from datetime import date, timedelta
from fractions import Fraction
import pytest
from app.analysis.series import MAD_SCALE, run_length
from tests.analysis_samples import analyze, call_error, fixed_prober, snap
SQL = "SELECT DATE(order_date, 'weekday 0', '-6 days') AS week, COUNT(*) AS orders FROM orders GROUP BY 1"
#: 每周一的日期:最后一周是 2026-10-12(到 10-18,运行当天 10-21 之前已经过完)
LAST_MONDAY = date(2026, 10, 12)
def weeks(values, *, last=LAST_MONDAY):
start = last - timedelta(days=7 * (len(values) - 1))
return [[(start + timedelta(days=7 * i)).isoformat(), v] for i, v in enumerate(values)]
def series(rows, sql=SQL, cols=("week", "orders")):
return snap("Q1", sql, list(cols), rows, types={cols[0]: "text", cols[1]: "number"})
async def baseline(rows, **params):
params = {"query": "Q1", "period": "week", "measure": "orders", "label": "订单数", **params}
return await analyze("baseline", snaps=[series(rows)], **params)
async def trend(rows, **params):
params = {"query": "Q1", "period": "week", "measure": "orders", "label": "线上销售额", **params}
return await analyze("trend", snaps=[series(rows)], **params)
NOISE = [100, 102, 98, 101, 99, 103, 100, 97, 102, 101, 99, 100, 98, 103, 101, 100, 99, 102, 100]
# --------------------------------------------------------------------------
# 波动基线
# --------------------------------------------------------------------------
async def test_level_basis_with_hand_computed_median_and_mad():
rows = [[i + 1, v] for i, v in enumerate([10, 12, 11, 13, 9, 10, 12, 11, 16])]
result = await baseline(rows, basis="level")
# 历史中位数 11,MAD 1,σ̂ = 1.4826,z = (16 − 11) / 1.4826
values = result["values"]
assert values["median"] == "11" and values["n_history"] == "8" and values["rank"] == "1"
assert Fraction(values["z"]) == pytest.approx(Fraction(5) / MAD_SCALE)
assert values["band_lo"] == "8.0348" and values["band_hi"] == "13.9652"
assert result["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "up"}
assert result["phrase"].startswith("订单数本期为 16,明显超出过去 8 期的常规波动范围(8.03 至 13.97)")
assert any(c["code"] == "record_high" for c in result["caveats"])
assert result["params"]["grain"] == "seq" and result["params"]["lag"] == 1
assert result["settings"] == {"k_warn": 2.0, "k_alert": 3.0, "min_history": 8, "max_history": 52}
assert result["table"]["rows"][-1][-1] == "本期"
async def test_change_basis_within_outside_and_far_outside():
probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")})
within = await analyze("baseline", snaps=[series(weeks(NOISE + [101]))], probe=probe, query="Q1", period="week",
measure="orders", label="订单数")
assert within["verdict"]["code"] == "within_band"
assert within["phrase"].startswith("订单数本期变化 +1.0%,在过去 18 周的常规波动范围内(")
assert within["params"]["today"] == "2026-10-21" and within["params"]["grain"] == "week"
assert within["probes"][0]["table"] == "orders" and within["probes"][0]["column"] == "order_date"
assert not any(c["code"] == "comparability" for c in within["caveats"])
outside = await analyze("baseline", snaps=[series(weeks(NOISE + [107]))], probe=probe, query="Q1",
period="week", measure="orders")
assert outside["verdict"]["code"] == "outside_band" and "超出过去 18 周" in outside["phrase"]
far = await analyze("baseline", snaps=[series(weeks(NOISE + [70]))], probe=probe, query="Q1", period="week",
measure="orders", label="订单数")
assert far["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "down"}
assert far["phrase"].startswith("订单数本期变化 -30.0%,明显超出过去 18 周的常规波动范围")
assert any(c["code"] == "record_low" and c["text"] == "本期的订单数是过去 18 周里最低的" for c in far["caveats"])
async def test_without_a_table_range_completeness_is_said_to_be_unverified_but_not_in_the_phrase():
result = await baseline(weeks(NOISE + [101]))
note = next(c for c in result["caveats"] if c["code"] == "comparability")
assert note["text"] == "没能核对本期的数据是否到齐(查不到表里这一列的日期范围)"
assert note["text"] not in result["phrase"]
async def test_flat_history_has_no_spread():
rows = [[i + 1, 50] for i in range(10)]
same = await baseline(rows, basis="level")
assert same["verdict"]["code"] == "within_band" and same["values"]["z"] is None
assert same["phrase"] == "订单数本期为 50,和过去 9 期一样(每期都是 50)"
jump = await baseline(rows[:-1] + [[10, 51]], basis="level")
assert jump["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "up"}
assert jump["phrase"].startswith("订单数本期为 51,明显超出过去 9 期的常规波动范围(每期都是 50)")
async def test_short_history_partial_target_gaps_and_zero_bases_are_refused():
short = await baseline(weeks([100, 101, 99, 102, 98, 100]))
# 6 周:第一周没有上一周可比,本期不算历史,历史的变化只有 4 个
assert short["reason_code"] == "history_short" and short["reason"] == "只有 4 期历史,至少要 8 期"
assert short["phrase"] == "波动基线没有做成:只有 4 期历史,至少要 8 期"
# 最后一周是 10-19 到 10-25:运行当天 10-21 还没过完
partial = await baseline(weeks(NOISE + [60], last=date(2026, 10, 19)))
assert partial["reason_code"] == "partial_target"
assert partial["reason"] == "本期数据只到 2026-10-20,还差 5 天,和完整的周比没有意义"
# 周过完了,但表里的数据只到 10-17
probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-17")})
late = await analyze("baseline", snaps=[series(weeks(NOISE + [60]))], probe=probe, query="Q1", period="week",
measure="orders")
assert late["reason"] == "本期数据只到 2026-10-17,还差 1 天,和完整的周比没有意义"
gap_rows = weeks(NOISE + [100, 100])
del gap_rows[-2] # 本期的上一周没有
gap = await baseline(gap_rows)
assert gap["reason_code"] == "target_gap"
zero = await baseline(weeks(NOISE + [0, 10]))
assert zero["reason_code"] == "zero_base"
unreadable = await baseline([["上周", 1], ["本周", 2]])
assert unreadable["reason_code"] == "period_unreadable" and "上周" in unreadable["reason"]
async def test_daily_series_compare_with_the_same_weekday_last_week():
start = date(2026, 7, 1)
rows = [[(start + timedelta(days=i)).isoformat(), 100 + (i % 7) * 10] for i in range(100)]
result = await analyze("baseline", snaps=[series(rows, cols=("day", "orders"))], query="Q1", period="day",
measure="orders", label="订单数", check=True)
assert result["params"]["grain"] == "day" and result["params"]["lag"] == 7
# 星期效应:和上周同一天比,每天的变化都是 0
assert result["values"]["median"] == "0" and result["verdict"]["code"] == "within_band"
assert result["phrase"].startswith("订单数比上周同一天变化 0.0%,和过去 52 天一样")
yoy = await baseline(weeks(NOISE * 3), lag=52) # 57 周:能和 52 周前比的历史只有 4 期
assert yoy["reason_code"] == "history_short" and yoy["reason"] == "只有 4 期历史,至少要 8 期"
assert (await baseline(weeks(NOISE * 4), lag=52))["status"] == "ok"
assert (await call_error("baseline", snaps=[series(weeks(NOISE))], query="Q1", period="week", measure="orders",
lag=0)).code == "bad_lag"
async def test_duplicate_periods_and_given_targets():
rows = weeks(NOISE + [101])
error = await call_error("baseline", snaps=[series(rows + [rows[-1]])], query="Q1", period="week",
measure="orders")
assert error.code == "duplicate_rows" and "在 SQL 里按 week 汇总" in error.message
earlier = await baseline(rows, target=rows[-3][0])
assert earlier["params"]["target"] == rows[-3][0] and earlier["values"]["n_history"] == "16"
missing = await call_error("baseline", snaps=[series(rows)], query="Q1", period="week", measure="orders",
target="2020-01-06")
assert missing.code == "period_not_found"
async def test_iso_week_labels_and_months():
labels = [[f"2026-W{w:02d}", v] for w, v in zip(range(22, 42), NOISE + [101])]
result = await baseline(labels)
assert result["params"]["grain"] == "week" and result["verdict"]["code"] == "within_band"
months = [[f"2025-{m:02d}", 100 + m] for m in range(1, 13)] + [[f"2026-{m:02d}", 112 + m] for m in range(1, 10)]
monthly = await baseline(months)
assert monthly["params"]["grain"] == "month" and "过去 19 个月" in monthly["phrase"]
# --------------------------------------------------------------------------
# 趋势
# --------------------------------------------------------------------------
async def test_rising_and_falling_trends_with_run_length():
rising = await trend(weeks([100 + 3 * i + (i % 3) for i in range(12)]))
assert rising["verdict"] == {"code": "rising", "subjects": [], "direction": "up"}
assert rising["values"]["n"] == "12" and rising["values"]["run_length"] == "11"
assert rising["phrase"].startswith("过去 12 周线上销售额呈上升趋势(每周约 +")
assert ",已连续 11 周上升" in rising["phrase"]
assert rising["settings"] == {"trend_min_points": 6, "trend_alpha": 0.05, "max_history": 52}
falling = await trend(weeks([200, 196, 197, 190, 185, 186, 180, 181, 175, 170]))
assert falling["verdict"]["code"] == "falling"
assert falling["values"]["run_length"] == "2" and "已连续" not in falling["phrase"]
async def test_no_trend_and_exact_small_sample_p():
flat = await trend(weeks([100, 104, 98, 103, 97, 101, 99, 102]))
assert flat["verdict"] == {"code": "no_trend", "subjects": [], "direction": None}
assert flat["phrase"] == "过去 8 周线上销售额没有明显的上升或下降趋势"
six = await trend(weeks([1, 2, 3, 4, 5, 6]))
# 没有并列、n ≤ 10:精确分布,单调的 6 期双侧 p = 2 / 6!
assert Fraction(six["values"]["p"]) == pytest.approx(Fraction(2, 720))
assert six["verdict"]["code"] == "rising" and six["values"]["slope"] == "1"
def test_run_length_counts_consecutive_changes_from_the_end():
assert run_length([Fraction(v) for v in (1, 2, 3, 2, 3, 4, 5)]) == (3, "up")
assert run_length([Fraction(v) for v in (5, 4, 3)]) == (2, "down")
assert run_length([Fraction(v) for v in (1, 2, 2)]) == (0, None)
async def test_trend_needs_enough_consecutive_points():
rows = weeks([100 + i for i in range(10)])
del rows[-4] # 最后连着的只有 3 期
gap = await trend(rows)
assert gap["reason_code"] == "series_gap" and gap["reason"] == "最后连着的只有 3 期,再往前缺期(至少要连着的 6 期)"
short = await trend(weeks([1, 2, 3, 4]))
assert short["reason_code"] == "history_short" and short["reason"] == "只有 4 期,至少要 6 期"
partial = await trend(weeks([100 + i for i in range(10)], last=date(2026, 10, 19)))
assert partial["reason_code"] == "partial_target"
async def test_daily_trends_warn_about_weekday_effects():
start = date(2026, 8, 1)
rows = [[(start + timedelta(days=i)).isoformat(), 100 + i] for i in range(30)]
result = await analyze("trend", snaps=[series(rows, cols=("day", "orders"))], query="Q1", period="day",
measure="orders")
assert result["verdict"]["code"] == "rising"
assert any(c["code"] == "weekday_effect" and c["text"] == "日数据有星期效应,看趋势建议按周汇总"
for c in result["caveats"])
assert "日数据有星期效应" in result["phrase"]
+184
View File
@@ -0,0 +1,184 @@
"""分析用的统计函数(app/analysis/stats.py):用公认的表值钉住,外加边界——并列、n 很小、p 接近 0 / 1、t 自由度很大趋近正态。
Wilson 从 eval/stats 搬过来,评测引用的是同一个函数、结果和以前一样。"""
from __future__ import annotations
import itertools
import math
import random
from fractions import Fraction
import pytest
from app.analysis import stats
from app.eval import stats as eval_stats
def test_normal_quantiles_and_distribution_match_the_tables():
assert stats.normal_ppf(0.975) == pytest.approx(1.959964, abs=1e-6)
assert stats.normal_ppf(0.975) == pytest.approx(stats.Z95, rel=1e-15)
assert stats.normal_ppf(0.5) == 0.0
assert stats.normal_ppf(0.95) == pytest.approx(1.644854, abs=1e-6)
assert stats.normal_ppf(0.995) == pytest.approx(2.575829, abs=1e-6)
assert stats.normal_ppf(1e-10) == pytest.approx(-6.361341, abs=1e-6) # 很靠近 0 的尾巴
assert stats.normal_ppf(0) == -math.inf and stats.normal_ppf(1) == math.inf
for p in (1e-12, 0.001, 0.02425, 0.3, 0.7, 0.97575, 0.999, 1 - 1e-9):
assert stats.normal_cdf(stats.normal_ppf(p)) == pytest.approx(p, rel=1e-9)
assert stats.normal_cdf(1.959963984540054) == pytest.approx(0.975, abs=1e-15)
assert stats.normal_two_sided(1.959963984540054) == pytest.approx(0.05, abs=1e-15)
def test_t_quantiles_match_the_tables():
# 双侧 0.05 的临界值(t 分布表)
table = {1: 12.706205, 2: 4.302653, 5: 2.570582, 10: 2.228139, 30: 2.042272, 100: 1.983972}
for df, value in table.items():
assert stats.t_ppf(0.975, df) == pytest.approx(value, abs=1e-6), df
assert stats.t_two_sided(2.228139, 10) == pytest.approx(0.05, abs=1e-6)
# 自由度 2 有闭式解:F(t) = 1/2 + t / (2√(2 + t²))
for t in (0.3, 1.5, 4.0, 25.0):
assert stats.t_cdf(t, 2) == pytest.approx(0.5 + t / (2 * math.sqrt(2 + t * t)), rel=1e-12)
assert stats.t_cdf(0.0, 7) == 0.5 and stats.t_two_sided(0.0, 7) == 1.0
assert stats.t_cdf(-2.0, 10) == pytest.approx(1 - stats.t_cdf(2.0, 10), rel=1e-12)
def test_t_with_a_huge_df_approaches_the_normal():
assert stats.t_ppf(0.975, 1e6) == pytest.approx(stats.Z95, abs=1e-5)
assert stats.t_ppf(0.975, 1e9) == pytest.approx(stats.Z95, abs=1e-9)
assert stats.t_two_sided(1.96, 5e6) == pytest.approx(stats.normal_two_sided(1.96), abs=1e-6)
# 非整数自由度(Welch)也行
assert stats.t_ppf(0.975, 41.784) == pytest.approx(2.018, abs=1e-3)
def test_regularized_incomplete_beta():
assert stats.betainc(2, 3, 0.4) == pytest.approx(0.5248, abs=1e-12) # 多项式,手算
assert stats.betainc(0.5, 0.5, 0.5) == pytest.approx(0.5, abs=1e-12)
assert stats.betainc(1, 1, 0.3) == pytest.approx(0.3, abs=1e-12)
assert stats.betainc(3, 2, 0.0) == 0.0 and stats.betainc(3, 2, 1.0) == 1.0
def test_wilson_moved_here_is_the_same_function_the_eval_uses():
assert eval_stats.wilson is stats.wilson
assert eval_stats.Z95 == stats.Z95
assert stats.wilson(8, 10) == (0.4902, 0.9433)
assert stats.wilson(0, 0) is None
assert stats.wilson(0, 5)[0] == 0.0 and stats.wilson(5, 5)[1] == 1.0
low, high = stats.wilson(8, 10, places=None)
assert round(low, 4) == 0.4902 and round(high, 4) == 0.9433 and low != 0.4902
def test_newcombe_matches_the_published_example():
# Newcombe(1998)的例子:56/70 对 48/80,混合 score 法(方法 10)0.0524 到 0.3339
low, high = stats.newcombe(56, 70, 48, 80)
assert low == pytest.approx(0.0524, abs=1e-4) and high == pytest.approx(0.3339, abs=1e-4)
assert stats.newcombe(1, 0, 2, 5) is None
def test_pooled_z_and_min_expected():
z, p = stats.pooled_z(56, 70, 48, 80)
assert z == pytest.approx(2.6502, abs=1e-4) and p == pytest.approx(0.00805, abs=1e-5)
assert stats.pooled_z(0, 10, 0, 20) == (0.0, 1.0) # 两边全不是:没有差异可言
assert stats.pooled_z(3, 0, 1, 5) is None
assert stats.min_expected(2, 40, 5, 230) == pytest.approx(7 * 40 / 270)
def test_fisher_exact_two_sided():
# Fisher 的品茶实验
assert stats.fisher_exact(3, 1, 1, 3) == pytest.approx(0.4857142857, abs=1e-9)
assert stats.fisher_exact(1, 9, 11, 3) == pytest.approx(0.002759456, abs=1e-8)
assert stats.fisher_exact(0, 10, 10, 0) == pytest.approx(2 / math.comb(20, 10), rel=1e-9) # p 接近 0
assert stats.fisher_exact(5, 5, 5, 5) == pytest.approx(1.0) # p 接近 1
assert stats.fisher_exact(0, 0, 0, 0) == 1.0
def test_welch_t_test():
test = stats.welch(10, 4, 20, 8, 9, 25)
# t = 2 / √(4/20 + 9/25) = 2.6726;自由度按 Welch–Satterthwaite
assert test.t == pytest.approx(2.672612, abs=1e-6)
assert test.df == pytest.approx(0.56 ** 2 / (0.2 ** 2 / 19 + 0.36 ** 2 / 24), rel=1e-12)
assert test.p == pytest.approx(stats.t_two_sided(test.t, test.df))
assert stats.welch(1, 1, 1, 2, 1, 5) is None # 一组只有一个样本:方差算不了
assert stats.welch(5, 0, 3, 5, 0, 4).p == 1.0 # 没有方差、均值一样
assert stats.welch(5, 0, 3, 6, 0, 4).p == 0.0
low, high = stats.welch_interval(10, 4, 20, 8, 9, 25)
assert low < 2 < high and (low + high) / 2 == pytest.approx(2)
def test_holm_step_down():
assert stats.holm([0.01, 0.04, 0.03, 0.005]) == pytest.approx([0.03, 0.06, 0.06, 0.02])
assert stats.holm([0.01, None, 0.5]) == [0.02, None, 0.5]
assert stats.holm([0.6, 0.9]) == [1.0, 1.0] # 不超过 1
assert stats.holm([]) == []
def _exact_two_sided(values: list[int]) -> Fraction:
"""暴力枚举全部排列,算 Mann–Kendall 的精确双侧 p(对照用)。"""
def s_of(seq: tuple[int, ...]) -> int:
return sum((b > a) - (b < a) for i, a in enumerate(seq) for b in seq[i + 1:])
observed = abs(s_of(tuple(values)))
perms = list(itertools.permutations(values))
return Fraction(sum(1 for p in perms if abs(s_of(p)) >= observed), len(perms))
def test_mann_kendall_small_samples_use_the_exact_distribution():
mk = stats.mann_kendall([1, 2, 3, 4])
assert mk.exact and mk.s == 6 and mk.p == Fraction(1, 12) # 4! 里只有两头两个排列
assert stats.mann_kendall([1, 2, 3, 4, 5]).p == Fraction(1, 60)
for values in ([3, 1, 4, 2, 6, 5], [1, 3, 2, 5, 4, 7, 6], [2, 1, 4, 3, 6, 5, 8]):
assert stats.mann_kendall(values).p == _exact_two_sided(values)
# n = 10 的单调序列:2 / 10!
assert stats.mann_kendall(list(range(10))).p == Fraction(2, math.factorial(10))
def test_mann_kendall_with_ties_or_more_points_uses_the_corrected_normal_approximation():
values = [1, 2, 2, 3, 4, 5, 5, 6, 7, 8, 9, 10]
mk = stats.mann_kendall(values)
n = len(values)
var = (n * (n - 1) * (2 * n + 5) - 2 * (2 * 1 * 9)) / 18 # 两组各两个并列
assert not mk.exact and mk.var == pytest.approx(var)
assert mk.z == pytest.approx((mk.s - 1) / math.sqrt(var)) # 连续性校正
assert mk.p == pytest.approx(stats.normal_two_sided(mk.z))
tied = stats.mann_kendall([4, 4, 4, 4, 4, 4]) # 全是并列:方差为 0
assert tied.s == 0 and tied.p == 1.0
small_tied = stats.mann_kendall([1, 1, 2, 3]) # n ≤ 10 但有并列:也走正态近似
assert not small_tied.exact
def test_theil_sen_median_mad_and_rank():
xs = [Fraction(i) for i in range(4)]
assert stats.theil_sen(xs, [Fraction(1), Fraction(3), Fraction(2), Fraction(6)]) == Fraction(19, 12)
assert stats.theil_sen([Fraction(0)], [Fraction(1)]) is None
assert stats.median([Fraction(3), Fraction(1), Fraction(2)]) == 2
assert stats.median([Fraction(1), Fraction(2), Fraction(3), Fraction(10)]) == Fraction(5, 2)
assert stats.median([]) is None
assert stats.mad([Fraction(v) for v in (1, 1, 2, 2, 4, 6, 9)]) == 1
assert stats.rank_desc(5, [1, 7, 5, 9]) == 3 and stats.rank_desc(10, [1, 2]) == 1
def test_theil_sen_on_many_points_matches_the_plain_definition():
rng = random.Random(9)
for n in (2, 3, 11, 60):
xs = [Fraction(i) for i in range(n)]
ys = [Fraction(rng.randint(-500, 500), rng.choice([1, 2, 3, 7, 100])) for _ in range(n)]
slopes = sorted((ys[j] - ys[i]) / (xs[j] - xs[i]) for i in range(n) for j in range(i + 1, n))
m = len(slopes)
expected = slopes[m // 2] if m % 2 else (slopes[m // 2 - 1] + slopes[m // 2]) / 2
assert stats.theil_sen(xs, ys) == expected
assert stats.theil_sen([0.0, 1.0, 2.0], [1.0, 3.0, 2.0]) == 0.5 # 浮点照样算
# 浮点分不开的斜率:精确地比
tiny = Fraction(1, 10 ** 25)
ys = [Fraction(0), Fraction(1) + tiny, Fraction(2) - tiny, Fraction(3)]
xs = [Fraction(i) for i in range(4)]
slopes = sorted((ys[j] - ys[i]) / (xs[j] - xs[i]) for i in range(4) for j in range(i + 1, 4))
assert stats.theil_sen(xs, ys) == (slopes[2] + slopes[3]) / 2
def test_the_fast_median_of_many_values_is_still_exact():
rng = random.Random(5)
for size in (513, 514, 2001):
# 浮点分不开的值:1/3 加上远小于浮点精度的扰动
values = [Fraction(1, 3) + Fraction(rng.randint(-3, 3), 10 ** 30) for _ in range(size)]
values += [Fraction(rng.randint(-1000, 1000), rng.randint(1, 260)) for _ in range(size)]
ordered = sorted(values)
n = len(ordered)
expected = ordered[n // 2] if n % 2 else (ordered[n // 2 - 1] + ordered[n // 2]) / 2
assert stats.median(values) == expected