diff --git a/backend/app/analysis/__init__.py b/backend/app/analysis/__init__.py index a4798d73..67c4ec37 100644 --- a/backend/app/analysis/__init__.py +++ b/backend/app/analysis/__init__.py @@ -8,14 +8,24 @@ - recompute：按结论里记下的 params、settings、probes 和输入快照（按 sha256 取回）重算一遍，不现查；和封存的用 contract.same 比。 -契约（名字、形状、缺省阈值）在 contract.py。 +契约（名字、形状、缺省阈值）在 contract.py。各算子在 decompose.py（变化拆解）、series.py（波动基线、趋势）、differ.py +（差异检验）、comparable.py（可比性检查）；统计在 stats.py，判定语和数的写法在 phrases.py，现查在 probe.py。 + +**一份结论只由 inputs（快照里的格）、probes、params、settings、method 决定**：run 时从 context 读到的东西（运行当天、目录 +里的度量类型和叫法、现查的范围）都记进结论——运行当天、目录事实在 params（today、catalog），现查结果在 probes——复算只读 +记下的。同样的输入两次 run 逐字节相同（现查结果相同时）；recompute 和 run 逐字段相同。 + +**参数**（AnalysisRequest.params；列名不分大小写，按快照认）见各算子文件头；params 里的 catalog、today、depth 由系统填， +调用方写了也不收。 """ from __future__ import annotations from collections.abc import Callable, Mapping +from datetime import date, datetime from typing import Any -from app.analysis.contract import AnalysisRequest, AnalysisResult, Context, InputSnapshot +from app.analysis.contract import (MODES, OPS, AnalysisRequest, AnalysisResult, CallError, Context, InputSnapshot, + effective_settings) #: 按调用里写的名字取回输入快照；没有这个名字返回 None Resolver = Callable[[str], InputSnapshot | None] @@ -23,16 +33,144 @@ Resolver = Callable[[str], InputSnapshot | None] ParentResolver = Callable[[str], AnalysisResult | None] +def _op_mode(request: Mapping[str, Any]) -> tuple[str, str | None]: + from app.analysis.phrases import OP_NAMES + + op = request.get("op") + if op not in OPS: + raise CallError("unknown_op", f"不认识的分析：{op}。能做的有{'、'.join(OP_NAMES[o] for o in OPS)}") + modes = MODES.get(op) + mode = request.get("mode") + if modes is None: + return op, None + if mode not in modes: + raise CallError("unknown_mode", f"{OP_NAMES[op]}要写模式：{' / '.join(modes)}") + return op, mode + + +def _compute(op: str, mode: str | None, raw: Mapping[str, Any], env: Any) -> AnalysisResult: + from app.analysis import comparable, decompose, differ, series + + if op == "decompose": + return {"additive": decompose.additive, "ratio": decompose.ratio, "factor": decompose.factor}[mode](raw, env) # type: ignore[index] + if op == "baseline": + return series.baseline(raw, env) + if op == "trend": + return series.trend(raw, env) + if op == "differ": + return differ.differ(raw, env) + return comparable.comparable(raw, env) + + +def _env(op: str, mode: str | None, settings: Mapping[str, Any], inputs: Any, facts: Any, book: Any, + today: date | None, parents: ParentResolver | None, hypothesis: Any) -> Any: + from app.analysis.common import Env + + return Env(op=op, mode=mode, settings=settings, inputs=inputs, facts=facts, book=book, today=today, + parents=parents, hypothesis=hypothesis if isinstance(hypothesis, str) else None) + + async def run(request: AnalysisRequest, resolve: Resolver, *, settings: Mapping[str, Any], context: Context, parents: ParentResolver | None = None) -> AnalysisResult: """算一份分析结论。settings 是 contract.effective_settings 算好的阈值。""" - raise NotImplementedError("analysis.run：第 1 波 W1-a 实现") + from app.analysis.common import DERIVED_PARAMS + from app.analysis.facts import Facts + from app.analysis.inputs import Inputs + from app.analysis.probe import MAX_ROUNDS, Book, fetch + + op, mode = _op_mode(request) + raw = {k: v for k, v in (request.get("params") or {}).items() if k not in DERIVED_PARAMS} + full = effective_settings(settings) + prober = context.get("probe") + today = context.get("today") or date.today() + if isinstance(today, datetime): + today = today.date() + + def dialect(name: str, snap: Mapping[str, Any]) -> str | None: + given = snap.get("dialect") + if isinstance(given, str) and given: + return given + lookup = getattr(prober, "dialect", None) + return lookup(str(snap.get("source") or "")) if lookup is not None else None + + known: dict[tuple[str, ...], Any] = {} + for rounds in range(MAX_ROUNDS + 1): + book = Book(known) + env = _env(op, mode, full, Inputs(resolve, dialect), Facts(catalogs=context.get("catalogs")), book, today, + parents, request.get("hypothesis")) + result = _compute(op, mode, raw, env) + missing = book.missing + if not missing or rounds == MAX_ROUNDS: + return result + known.update(await fetch(book, context)) + return result # pragma: no cover - 循环里一定返回 def recompute(result: AnalysisResult, load: Callable[[str], InputSnapshot | None], *, load_result: Callable[[str], AnalysisResult | None] | None = None) -> AnalysisResult: - """复算：load 按 sha256 取回输入快照，load_result 按 sha256 取回 parent 那份结论。""" - raise NotImplementedError("analysis.recompute：第 1 波 W1-a 实现") + """复算：load 按 sha256 取回输入快照，load_result 按 sha256 取回 parent 那份结论。用结论里记下的 params（含运行当天、 + 目录事实）、settings、probes，不现查、不读目录。取不回输入时抛 CallError（code input_missing）；方法升版后调用不再成立 + 时同样抛 CallError（照 run 的说法）。和封存的那份用 contract.same 比。""" + from app.analysis.facts import Facts + from app.analysis.inputs import _CELL, Inputs + from app.analysis.probe import Book + + op, mode = _op_mode(result) + params = dict(result.get("params") or {}) + settings = effective_settings(result.get("settings")) + names: dict[str, tuple[str | None, str | None, list[str]]] = {} + parents_at: dict[str, str] = {} + for item in result.get("inputs") or []: + ref = str(item.get("ref") or "") + if item.get("role") == "parent": + parents_at[ref] = str(item.get("artifact") or "") + continue + cell = _CELL.match(ref) if "row" in item else None + name = cell.group("name") if cell else ref + names.setdefault(name, (item.get("artifact"), item.get("dialect"), list(item.get("masked") or []))) + + def resolve(name: str) -> InputSnapshot | None: + artifact, _, masked = names.get(name, (None, None, [])) + found = load(artifact) if artifact else None + if not isinstance(found, Mapping): + raise CallError("input_missing", f"复算取不回 {name} 的查询快照") + return {**found, "artifact": artifact, "masked": masked} # type: ignore[typeddict-item] + + def dialect(name: str, snap: Mapping[str, Any]) -> str | None: + return names.get(name, (None, None, []))[1] + + def parents(name: str) -> AnalysisResult | None: + artifact = parents_at.get(name) + found = load_result(artifact) if artifact and load_result is not None else None + if not isinstance(found, Mapping): + raise CallError("input_missing", f"复算取不回上一层的分析结论 {name}") + return found # type: ignore[return-value] + + today = date.fromisoformat(params["today"]) if isinstance(params.get("today"), str) else None + env = _env(op, mode, settings, Inputs(resolve, dialect), Facts(recorded=params.get("catalog") or {}), + Book(result.get("probes") or []), today, parents, result.get("hypothesis")) + raw = {k: v for k, v in params.items() if k not in ("catalog", "today", "depth")} + return _compute(op, mode, raw, env) -__all__ = ["ParentResolver", "Resolver", "recompute", "run"] +def exhausted(settings: Mapping[str, Any]) -> CallError: + """本运行分析次数用完（max_calls）的调用错误：入口数着次数，到了上限交回它，不再算。""" + limit = int(effective_settings(settings)["max_calls"]) + return CallError("max_calls", f"这次运行的分析次数已经用完（最多 {limit} 次）：用已有的分析结论写，不再新做分析") + + +def make_prober(*args: Any, **kwargs: Any) -> Any: + """见 app/analysis/probe.make_prober。""" + from app.analysis.probe import make_prober as make + + return make(*args, **kwargs) + + +def comparable_reasons(result: Mapping[str, Any]) -> list[str]: + """可比性结论里 fail、warn 的原因，一句一个（口径卡指标的 comparability.reasons）。""" + from app.analysis.comparable import reasons + + return reasons(result) + + +__all__ = ["ParentResolver", "Resolver", "comparable_reasons", "exhausted", "make_prober", "recompute", "run"] diff --git a/backend/app/analysis/common.py b/backend/app/analysis/common.py new file mode 100644 index 00000000..bb323774 --- /dev/null +++ b/backend/app/analysis/common.py @@ -0,0 +1,136 @@ +"""算子共用的：一次计算的环境（输入、目录事实、现查簿、阈值）、结论的组装、几样参数的规范化。 + +每个算子是 (raw params, Env) → AnalysisResult 的纯函数：只经 Env 取输入快照（Inputs）、目录事实（Facts）、现查结果 +（Book），阈值只来自 Env.settings。run 和 recompute 只是装 Env 的方式不同（见 app/analysis/__init__.py）。 +""" +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import date +from fractions import Fraction +from typing import Any + +from app.analysis import phrases as ph +from app.analysis.contract import METHODS, AnalysisResult, CallError +from app.analysis.facts import Facts +from app.analysis.inputs import Inputs +from app.analysis.numbers import to_fraction +from app.analysis.probe import Book + +#: 结论的 params 里由系统填、不收调用方的几项：目录事实、运行当天、下钻的层数 +DERIVED_PARAMS = frozenset({"catalog", "today", "depth"}) + + +@dataclass +class Env: + op: str + mode: str | None + settings: Mapping[str, Any] + inputs: Inputs + facts: Facts + book: Book + today: date | None + parents: Callable[[str], Mapping[str, Any] | None] | None + hypothesis: str | None + + def threshold(self, key: str) -> Fraction: + value = to_fraction(self.settings[key]) + assert value is not None, key + return value + + +def label_for(env: Env, raw: Any, snap: Any, column: str | None, *, default: str | None = None) -> str: + """判定语里的度量叫法：agent 给的 label → 目录里的中文名 → default → 列名，依次取第一个消毒后不为空的（agent 给了能用的 + 就不去查目录）。""" + if ph.clean_label(raw): + return ph.clean_label(raw) + catalog = env.facts.label(snap, column) if column else None + return ph.pick_label(catalog, default, column) + + +def column_label(env: Env, snap: Any, column: str) -> str: + """维度、因子这类列的叫法：目录里的中文名，没有就是列名（都消毒）。""" + return ph.pick_label(env.facts.label(snap, column), column) + + +def today_param(env: Env) -> str: + """要判本期完不完整的算子把运行当天记进 params（复算用记下的那一天）。""" + day = env.today or date.today() + env.today = day + return day.isoformat() + + +def build(env: Env, params: dict[str, Any], used: Sequence[str], *, status: str = "ok", + reason_code: str | None = None, reason: str | None = None, table: Mapping[str, Any] | None = None, + values: Mapping[str, Any] | None = None, verdict: Mapping[str, Any] | None = None, criterion: str, + phrase: str, caveats: Sequence[Mapping[str, str]] = ()) -> AnalysisResult: + """组装一份结论：形状严格按 contract.AnalysisResult，params 补上查过的目录事实。""" + record = env.facts.record() + out_params = {k: v for k, v in params.items() if v is not None} + if record: + out_params["catalog"] = record + return { + "op": env.op, + "mode": env.mode, + "method": METHODS[(env.op, env.mode)], + "params": out_params, + "settings": {k: env.settings[k] for k in used}, + "inputs": list(env.inputs.records), + "probes": env.book.used, + "status": status, # type: ignore[typeddict-item] + "reason_code": reason_code, + "reason": reason, + "table": dict(table) if table is not None else None, # type: ignore[typeddict-item] + "values": dict(values or {}), + "verdict": dict(verdict) if verdict is not None else None, # type: ignore[typeddict-item] + "criterion": criterion, + "phrase": phrase, + "caveats": [dict(c) for c in caveats], # type: ignore[misc] + "hypothesis": env.hypothesis, + } + + +def refuse(env: Env, params: dict[str, Any], used: Sequence[str], code: str, reason: str, *, + criterion: str, caveats: Sequence[Mapping[str, str]] = ()) -> AnalysisResult: + """算不了（status refused）：判定语是「{算子的叫法}没有做成：{原因}」。""" + return build(env, params, used, status="refused", reason_code=code, reason=reason, criterion=criterion, + phrase=ph.refused(env.op, reason), caveats=caveats) + + +def need(raw: Mapping[str, Any], key: str, what: str) -> Any: + """必填的参数；没写是调用错误。""" + value = raw.get(key) + if value is None or (isinstance(value, str) and not value.strip()) or value == []: + raise CallError("missing_param", f"没写{what}") + return value + + +def written_places(raw: Any) -> int: + """快照里一格写到第几位小数（headline 是取整过的，按它的书写精度容差）：数字文本按字面，浮点按最短写法，整数为 0。""" + if isinstance(raw, bool) or raw is None: + return 0 + text = repr(raw) if isinstance(raw, float) else str(raw).strip() + if "e" in text.lower(): + return 0 + return len(text.split(".", 1)[1]) if "." in text else 0 + + +def closes(total: Fraction, cell: Fraction, raw: Any) -> bool: + """各段之和（或合起来的比率）和总数那一格对得上：十进制逐位相等；总数取整过的，差在它书写精度的半个末位以内也算；浮点 + 算出来的差在最后几位的也算（相对误差 1e-9 以内，同 engine/compare._equal）。""" + if total == cell: + return True + gap = abs(total - cell) + if gap <= Fraction(1, 2) / Fraction(10) ** written_places(raw): + return True + scale = max(abs(total), abs(cell)) + return scale != 0 and gap <= scale * Fraction(1, 10 ** 9) + + +def sign_word(value: Fraction | float | int) -> str | None: + return "up" if value > 0 else "down" if value < 0 else None + + +__all__ = ["DERIVED_PARAMS", "Env", "build", "closes", "column_label", "label_for", "need", "refuse", "sign_word", + "today_param", "written_places"] diff --git a/backend/app/analysis/comparable.py b/backend/app/analysis/comparable.py new file mode 100644 index 00000000..c34abfce --- /dev/null +++ b/backend/app/analysis/comparable.py @@ -0,0 +1,540 @@ +"""可比性检查（细案 01 §6）：比两期之前先确认能比。三处用到：agent 单独调用（analysis_comparable）；变化拆解给了两份快照时 +先做；口径卡跨两份查询算环比、同比时自动做（入口经 run 调）。波动基线、趋势判本期完整用这里的 span_coverage。 + +五项检查，每项 pass / warn / fail / unknown（unknown 只写「核对不了」，不判不可比）： + +- 同口径：两份 SQL 一字不差 → pass；规范化（data/sql_semantics.normalize_sql）后只差字面量：把时间条件的界（data/coverage + 认出的时间谓词的界，整个表达式）和像日期的字面量换成同一个占位再比，一样就是只差日期 → pass，不一样就是比的两段不是 + 两期 → fail「两边除了日期，筛选的还不一样」；规范形式不同交给口径账本的 explain_sql_diff，等价 → pass，只差时间范围、排序 + → pass，否则 fail，写出差在哪些方面（只写方面，不写 SQL 原文里的值）。两份来自不同的数据源 fail。同一份快照里按期次分组 + 的天然同口径。 +- 窗口长度：用 data/coverage 的 filters 取业务日期列（目录的 business_date；没写就取唯一那一列时间谓词）上的区间，界是字面 + 量的直接解析，按运行当天现算的在库里算（Book 的 bound，记进 probes）；算两边天数。整月对整月、整季对整季算一样长。 + 不相等：度量是流量 fail，比率、存量、没标的 warn。按期次分组的：两期粒度相同，一样长。 +- 整期：窗口（或这一期）的终点和表里这一列的实际最大值（Book 的 range：现查，查不成退回剖析事实）、运行当天比；数据从 + 窗口起点之后才开始的也算不完整。不完整：流量 fail（「本期数据只到 2026-10-15，还差 3 天」），其余 warn；表的范围查不到时 + 只能按运行当天判，判不出的 unknown。 +- 星期构成：按天的窗口、天数不是 7 的倍数时数两边的周六、周日天数，不同 warn（「两段的周末天数不同：4 天对 6 天」）。整月、 + 整季、按周的不看。 +- 数据问题：两边查询结果上「查询区间里这张表没有数据」（快照的 no_data）fail，SQL 检查的错误级结果（快照的 checks，level + error）warn。 + +判定：有 fail → incomparable；只有 warn → conditional；其余 comparable（有核对不了的项照实写在判定语后面）。 +""" +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from datetime import date, timedelta +from typing import Any + +from app.analysis import periods as pd +from app.analysis import phrases as ph +from app.analysis.facts import Facts, date_source +from app.analysis.inputs import Snap +from app.analysis.probe import Book +from app.data.names import name_key + +CHECKS = ("same_query", "window", "complete", "weekdays", "data") +CHECK_NAMES = {"same_query": "同口径", "window": "窗口长度", "complete": "整期", "weekdays": "星期构成", + "data": "数据问题"} +STATUS_NAMES = {"pass": "通过", "warn": "提醒", "fail": "不通过", "unknown": "核对不了"} +#: explain_sql_diff 的方面 → 叫法 +_ASPECTS = {"time": "时间列", "time_range": "时间范围", "filter": "筛选条件", "aggregate": "聚合方式", + "expression": "选出来的列", "join": "关联", "tables": "查的表", "set_op": "合并方式", "group_by": "分组", + "having": "分组后的筛选", "limit": "取哪几行", "order": "排序", "parse": "写法", "other": "写法"} +#: 只差这几个方面的两份 SQL 仍是同一个量的两期 +_SAME_ASPECTS = frozenset({"time_range", "order"}) +#: 一项 SQL 检查的说明最多引多少字 +_CHECK_TEXT_MAX = 80 + + +@dataclass +class Check: + id: str + status: str = "unknown" + reasons: list[str] = field(default_factory=list) + base: str | None = None + target: str | None = None + + +@dataclass +class Span: + """一边的时间范围：start / end 是窗口（或这一期）的头尾（含）；table / column 是这段时间按的那一列（探查表的范围用）； + sql_window 是 SQL 时间条件在这一列上的区间（按期次分组时看它有没有截掉这一期）。""" + + label: str + snap: Snap + start: date | None = None + end: date | None = None + table: str | None = None + column: str | None = None + sql_window: Any = None + unit: str | None = None # 这一段是不是整月、整季、整年（calendar_unit），按期次时是粒度 + + +@dataclass +class Outcome: + checks: list[Check] + verdict: str + reasons: list[str] + unknown: list[str] + phrase: str + structure: str | None # 同口径不通过的原因（变化拆解据此不拆） + base: Span | None = None + target: Span | None = None + + def check(self, cid: str) -> Check: + return next(c for c in self.checks if c.id == cid) + + +# -------------------------------------------------------------------------- +# 同口径 +# -------------------------------------------------------------------------- + + +def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None: + """时间条件的界、像日期的字面量换成同一个占位之后的规范形式（文本 + 剩下的字面量）。解析不了为 None。""" + from sqlglot import exp + + from app.data import coverage + from app.data.sql_semantics import normalize_sql + + tree = coverage.parse_sql(sql, read) + if tree is None: + return None + for flt in coverage.filters(tree): + for _, node, _ in flt.conds: + if node.parent is not None: + node.replace(exp.Literal.string("__day__")) + for lit in list(tree.find_all(exp.Literal)): + if coverage.literal_bound(lit)[1] is not None: + lit.replace(exp.Literal.string("__day__")) + normalized = normalize_sql(tree.sql(dialect=read), read) + if normalized.parse_error: + return None + return normalized.text, normalized.literals + + +def same_query(base: Snap, target: Snap) -> Check: + from app.data.sql_semantics import explain_sql_diff, normalize_sql + + check = Check("same_query") + if base.source != target.source: + check.status, check.reasons = "fail", ["两份查询来自不同的数据源"] + return check + a, b = base.sql.strip(), target.sql.strip() + if a == b: + check.status = "pass" + return check + read = base.dialect or target.dialect + na, nb = normalize_sql(a, read), normalize_sql(b, read) + if na.parse_error or nb.parse_error: + check.reasons = ["SQL 解析不了"] + return check + if na.text == nb.text: + ua, ub = _undated(a, read), _undated(b, read) + if ua is None or ub is None: + return check + if ua == ub: + check.status = "pass" + else: + check.status, check.reasons = "fail", ["两边除了日期，筛选的还不一样"] + return check + try: + diff = explain_sql_diff(a, b, read) + except Exception: # noqa: BLE001 - 比不了就当核对不了 + return check + aspects = [d.aspect for d in diff.items] + if diff.equivalent or (aspects and set(aspects) <= _SAME_ASPECTS): + check.status = "pass" + return check + names = list(dict.fromkeys(_ASPECTS.get(x, "写法") for x in aspects if x not in _SAME_ASPECTS)) or ["写法"] + check.status, check.reasons = "fail", [f"两份查询的口径结构不同（差在{'、'.join(names)}）"] + return check + + +# -------------------------------------------------------------------------- +# 窗口 +# -------------------------------------------------------------------------- + + +def _bound_days(snap: Snap, conds: Sequence[tuple[str, Any, bool]], book: Book) -> dict[int, Any]: + from app.data import coverage + + days: dict[int, Any] = {} + for _, node, _ in conds: + literal, day = coverage.literal_bound(node) + if literal: + days[id(node)] = day + continue + try: + text = node.sql(dialect=snap.dialect) + except Exception: # noqa: BLE001 + days[id(node)] = None + continue + probe = book.bound(snap.source, text) + days[id(node)] = coverage.as_day(probe.get("value")) if probe else None + return days + + +def _filters_on(snap: Snap, measure: str | None, facts: Facts, + column: tuple[str, str] | None = None) -> tuple[Any, str | None]: + """SQL（收窄到度量那一项）里按业务日期列（或唯一那一列时间谓词，或给定的那一列）的时间条件：(那段条件, 认不出的原因)。""" + from app.data import coverage + + tree = coverage.parse_sql(snap.sql, snap.dialect) + if tree is None: + return None, "SQL 解析不了" + narrowed = coverage.narrowed(tree, measure, snap.dialect) if measure else tree + flts = coverage.filters(narrowed) + groups: dict[tuple[str, str], list[Any]] = {} + for flt in flts: + groups.setdefault((name_key(flt.table_name), name_key(flt.column_name)), []).append(flt) + if column is not None: + wanted = (name_key(column[0]), name_key(column[1])) + wanted_short = (name_key(column[0].rsplit(".", 1)[-1]), wanted[1]) + chosen = groups.get(wanted) or next((v for k, v in groups.items() + if (k[0].rsplit(".", 1)[-1], k[1]) == wanted_short), None) + if chosen is None: + return None, None + else: + if not groups: + return None, "SQL 里没有时间条件" + dated = [v for v in groups.values() + if (bd := facts.business_date(snap.source, v[0].table_name)) and name_key(bd) == name_key(v[0].column_name)] + if len(dated) == 1: + chosen = dated[0] + elif len(groups) == 1: + chosen = next(iter(groups.values())) + else: + return None, "时间条件不止一列，认不出按哪一列算" + if len({f.key for f in chosen}) > 1: + return None, "这一列上有好几段时间条件" + return chosen[0], None + + +def sql_span(label: str, snap: Snap, measure: str | None, facts: Facts, book: Book) -> tuple[Span, str | None]: + """一份查询的时间窗口（两份快照比两期时）。""" + from app.data import coverage + + span = Span(label, snap) + flt, why = _filters_on(snap, measure, facts) + if flt is None: + return span, why + if flt.column_name.lower() in snap.masked: + return span, "时间列是遮罩列" # 遮罩列不探表的范围 + span.table, span.column = flt.table_name, flt.column_name + window = coverage.window_of(flt.conds, _bound_days(snap, flt.conds, book)) + if window is None or window.start is None or window.end is None: + return span, "时间窗口没有两头" + span.start, span.end = window.start, window.end + span.unit = pd.calendar_unit(window.start, window.end) + return span, None + + +def period_span(label: str, snap: Snap, period: Any, item: pd.Period, grain: str, measure: str | None, + facts: Facts, book: Book) -> Span: + """按期次分组的一份快照里的一期：这一期覆盖的日子，加上 SQL 在期次那一列上的时间条件。""" + from app.data import coverage + + span = Span(label, snap, item.start, item.end, unit=grain) + found = date_source(snap.sql, period) + if found is None: + return span + span.table, span.column = found + if span.column.lower() in snap.masked: + span.table = span.column = None + return span + flt, _ = _filters_on(snap, measure, facts, column=found) + if flt is not None: + span.sql_window = coverage.window_of(flt.conds, _bound_days(snap, flt.conds, book)) + return span + + +def _range(span: Span, book: Book) -> tuple[bool, date | None, date | None, bool]: + """(查得到, 最早, 最晚, 一个值都没有)：表里这一列的实际范围。""" + from app.data import coverage + + if not (span.table and span.column): + return False, None, None, False + probe = book.range(span.snap.source, span.table, span.column) + if probe is None: + return False, None, None, False + lo = coverage.as_day(probe.get("min")) if probe.get("min") else None + hi = coverage.as_day(probe.get("max")) if probe.get("max") else None + empty = probe.get("basis") == "live" and probe.get("min") is None and probe.get("max") is None + return True, lo.day if lo else None, hi.day if hi else None, empty + + +def span_coverage(span: Span, today: date, book: Book) -> tuple[str, str | None]: + """这一段（窗口或一期）的数据齐不齐：(pass / short / unknown, 不齐的那一句)。没过完（终点不早于运行当天：当天的数据还没 + 到齐）、SQL 的时间条件截掉了一部分、表里的数据没到（最晚早于终点）或来得晚（最早晚于起点），都算不齐。""" + if span.start is None or span.end is None: + return "unknown", None + lo, hi = span.start, span.end + window = span.sql_window + if window is not None: + if window.start is not None: + lo = max(lo, window.start) + if window.end is not None: + hi = min(hi, window.end) + if span.end >= today: + hi = min(hi, today - timedelta(days=1)) + known, dmin, dmax, empty = _range(span, book) + if empty: + return "short", f"{span.label}在 {span.start.isoformat()} 到 {span.end.isoformat()} 没有数据" + if dmin is not None: + lo = max(lo, dmin) + if dmax is not None: + hi = min(hi, dmax) + if hi < lo: + return "short", f"{span.label}在 {span.start.isoformat()} 到 {span.end.isoformat()} 没有数据" + parts = [] + if hi < span.end: + parts.append(f"{span.label}数据只到 {hi.isoformat()}，还差 {(span.end - hi).days} 天") + if lo > span.start: + parts.append(f"{span.label}数据从 {lo.isoformat()} 才开始，缺前 {(lo - span.start).days} 天") + if parts: + return "short", "，".join(parts) + return ("pass", None) if known and dmax is not None else ("unknown", None) + + +def _days(span: Span) -> int | None: + return (span.end - span.start).days + 1 if span.start and span.end else None + + +def _shown(span: Span) -> str | None: + if span.start is None or span.end is None: + return None + if span.start == span.end: + return f"{span.start.isoformat()}（1 天）" + return f"{span.start.isoformat()} 到 {span.end.isoformat()}（{_days(span)} 天）" + + +# -------------------------------------------------------------------------- +# 五项检查和判定 +# -------------------------------------------------------------------------- + + +def _window_check(base: Span, target: Span, flow: bool, *, grouped: bool) -> Check: + check = Check("window", base=_shown(base), target=_shown(target)) + a, b = _days(base), _days(target) + if a is None or b is None: + return check + if a == b or grouped or (base.unit and base.unit == target.unit): + check.status = "pass" + return check + check.status = "fail" if flow else "warn" + check.reasons = [f"两段的天数不同：{a} 天对 {b} 天"] + return check + + +def _complete_check(sides: Sequence[Span], today: date, book: Book, flow: bool) -> Check: + check = Check("complete") + shorts, unknown = [], False + for span in sides: + status, reason = span_coverage(span, today, book) + if status == "short": + shorts.append(reason) + elif status == "unknown": + unknown = True + if span.label == "基期": + check.base = reason or STATUS_NAMES[status if status != "short" else "fail"] + else: + check.target = reason or STATUS_NAMES[status if status != "short" else "fail"] + if shorts: + check.status, check.reasons = ("fail" if flow else "warn"), [r for r in shorts if r] + elif not unknown: + check.status = "pass" + return check + + +def _weekday_check(base: Span, target: Span, *, grain: str | None) -> Check: + check = Check("weekdays") + a, b = _days(base), _days(target) + if a is None or b is None: + return check + if grain in ("week", "month", "quarter", "seq") or (a % 7 == 0 and b % 7 == 0) or \ + (base.unit and base.unit == target.unit and base.unit in ("month", "quarter", "year")): + check.status = "pass" + return check + wa, wb = pd.weekend_days(base.start, base.end), pd.weekend_days(target.start, target.end) # type: ignore[arg-type] + check.base, check.target = f"周末 {wa} 天", f"周末 {wb} 天" + if wa == wb: + check.status = "pass" + else: + check.status, check.reasons = "warn", [f"两段的周末天数不同：{wa} 天对 {wb} 天"] + return check + + +def _data_check(sides: Sequence[tuple[str, Snap]], measure: str | None) -> Check: + from app.data import coverage + + check = Check("data", status="pass") + fails, warns = [], [] + for label, snap in sides: + found = coverage.no_data_of(snap.data, measure) if measure else \ + next(iter(snap.data.get("no_data") or []), None) + if found: + fails.append(f"{label}查询的区间里这张表没有数据") + errors = [c for c in snap.data.get("checks") or [] if isinstance(c, Mapping) and c.get("level") == "error"] + if errors: + text = str(errors[0].get("message") or "").strip().replace("；", "，") + if len(text) > _CHECK_TEXT_MAX: + text = text[:_CHECK_TEXT_MAX - 1] + "…" + warns.append(f"{label}查询的 SQL 检查有错误：{text}" if text else f"{label}查询的 SQL 检查有错误") + if fails: + check.status = "fail" + elif warns: + check.status = "warn" + check.reasons = fails + warns + return check + + +def _decide(checks: list[Check], base: Span | None, target: Span | None) -> Outcome: + fails = [r for c in checks if c.status == "fail" for r in c.reasons] + warns = [r for c in checks if c.status == "warn" for r in c.reasons] + unknown = [CHECK_NAMES[c.id] for c in checks if c.status == "unknown"] + verdict = "incomparable" if fails else "conditional" if warns else "comparable" + reasons = fails + warns + structure = next((c.reasons[0] for c in checks if c.id == "same_query" and c.status == "fail" and c.reasons), None) + return Outcome(checks, verdict, reasons, unknown, ph.comparable_phrase(verdict, reasons, unknown), structure, + base, target) + + +def compare_queries(base: Snap, target: Snap, measure: str | None, facts: Facts, book: Book, today: date) -> Outcome: + """两份快照比两期（本周一份、上周一份）。measure 是两边的度量列（收窄 SQL 找时间条件、认度量类型）；没有为 None。""" + same = same_query(base, target) + b_span, _ = sql_span("基期", base, measure, facts, book) + t_span, _ = sql_span("本期", target, measure, facts, book) + flow = bool(measure) and facts.measure(target, measure)[0] == "flow" # type: ignore[arg-type] + checks = [same, _window_check(b_span, t_span, flow, grouped=False), + _complete_check([b_span, t_span], today, book, flow), _weekday_check(b_span, t_span, grain=None), + _data_check([("基期", base), ("本期", target)], measure)] + return _decide(checks, b_span, t_span) + + +def compare_periods(snap: Snap, period: str, base: pd.Period, target: pd.Period, grain: str, measure: str | None, + facts: Facts, book: Book, today: date) -> Outcome: + """同一份快照里按期次分组的两期：天然同口径；窗口按期次的粒度；整期看这两期各自齐不齐。""" + same = Check("same_query", status="pass") + b_span = period_span("基期", snap, period, base, grain, measure, facts, book) + t_span = period_span("本期", snap, period, target, grain, measure, facts, book) + flow = bool(measure) and facts.measure(snap, measure)[0] == "flow" # type: ignore[arg-type] + if grain == "seq": + checks = [same, Check("window"), Check("complete"), Check("weekdays"), _data_check([("", snap)], measure)] + else: + checks = [same, _window_check(b_span, t_span, flow, grouped=True), + _complete_check([b_span, t_span], today, book, flow), + _weekday_check(b_span, t_span, grain=grain), _data_check([("", snap)], measure)] + return _decide(checks, b_span, t_span) + + +def table_of(outcome: Outcome) -> dict[str, Any]: + """检查表（结论的 table）：检查项、结果、基期、本期、说明。""" + rows = [[CHECK_NAMES[c.id], STATUS_NAMES[c.status], c.base, c.target, "；".join(c.reasons) or None] + for c in outcome.checks] + return {"columns": ["check", "status", "base", "target", "note"], + "labels": {"check": "检查项", "status": "结果", "base": "基期", "target": "本期", "note": "说明"}, + "rows": rows} + + +def values_of(outcome: Outcome) -> dict[str, Any]: + out: dict[str, Any] = {c.id: c.status for c in outcome.checks} + for side, span in (("base", outcome.base), ("target", outcome.target)): + if span is not None and span.start and span.end: + out[f"{side}_start"] = span.start.isoformat() + out[f"{side}_end"] = span.end.isoformat() + out[f"{side}_days"] = str(_days(span)) + out[f"{side}_weekend"] = str(pd.weekend_days(span.start, span.end)) + return out + + +def reasons(result: Mapping[str, Any]) -> list[str]: + """一份可比性结论里 fail、warn 的原因，一句一个（口径卡指标的 comparability.reasons 用）。按检查表的说明拆开。""" + out: list[str] = [] + table = result.get("table") or {} + for row in table.get("rows") or []: + if len(row) >= 5 and row[1] in (STATUS_NAMES["fail"], STATUS_NAMES["warn"]) and row[4]: + out.extend(str(row[4]).split("；")) + return out + + +# -------------------------------------------------------------------------- +# 算子：analysis_comparable +# -------------------------------------------------------------------------- + + +def comparable(raw: Mapping[str, Any], env: Any) -> Any: + """可比性检查这个算子。参数：两份快照 base_query、target_query；或一份快照 query + period（期次列）+ base、target（期次 + 列里的两个值）、可选 grain。都可以写 measure（度量列：收窄 SQL 找时间条件、按目录认它是不是流量；不写时只有一列数值列 + 就用它）。""" + from app.analysis.common import build, need, today_param + from app.analysis.contract import CallError + from app.analysis.inputs import same_value + + params: dict[str, Any] = {} + two = raw.get("base_query") is not None or raw.get("target_query") is not None + if two: + base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号") + target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号") + if base is target: + from app.analysis.decompose import SAME_QUERY + + raise CallError("same_query", SAME_QUERY) + params.update(base_query=base.ref, target_query=target.ref) + else: + base = target = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号") + params["query"] = base.ref + measure = target.column(raw["measure"], "度量列") if raw.get("measure") is not None else _only_number(target) + if measure is not None and two: + base.column(measure, "度量列") + params["measure"] = measure + params["today"] = today_param(env) + if two: + env.inputs.use(base, "base", [measure] if measure else []) + env.inputs.use(target, "target", [measure] if measure else []) + outcome = compare_queries(base, target, measure, env.facts, env.book, env.today) + else: + period = target.column(need(raw, "period", "期次列"), "期次列") + grain = raw.get("grain") + if grain is not None and grain not in (*pd.GRAINS, "seq"): + raise CallError("bad_grain", "粒度只能写 day、week、month、quarter，或者不写（按期次值推断）") + params.update(period=period, base=need(raw, "base", "基期（期次列里的值）"), + target=need(raw, "target", "本期（期次列里的值）")) + i = target.index(period) + values = [row[i] if i < len(row) else None for row in target.rows] + rows = {side: [r for r, v in enumerate(values) if same_value(params[side], v)] for side in ("base", "target")} + for side, what in (("base", "基期"), ("target", "本期")): + if not rows[side]: + raise CallError("period_not_found", f"期次列「{period}」里没有「{params[side]}」：{what}要写 {target.ref} " + "里有的期次值") + env.inputs.use(target, "data", [period, *([measure] if measure else [])], rows["base"] + rows["target"]) + parsed = pd.parse(values, grain) + if parsed is None: + item_b = item_t = pd.Period(None, 0, None, None) + grain = "seq" + else: + items, grain = parsed + item_b, item_t = items[rows["base"][0]], items[rows["target"][0]] + params["grain"] = grain + outcome = compare_periods(target, period, item_b, item_t, grain, measure, env.facts, env.book, env.today) + caveats = [] + window = outcome.check("window") + if measure and window.status == "warn" and env.facts.measure(target, measure)[0] is None: + caveats.append(ph.caveat("measure_unlabeled", ph.unlabeled_text(measure, "窗口不等只作提醒"))) + verdict = {"code": outcome.verdict, "subjects": [], "direction": None} + return build(env, params, [], table=table_of(outcome), values=values_of(outcome), verdict=verdict, + criterion=ph.CRITERION_COMPARABLE, phrase=outcome.phrase, caveats=caveats) + + +def _only_number(snap: Snap) -> str | None: + """没写度量列时：快照里唯一一列（没遮罩的）数值列；没有或不止一列为 None。""" + found = [c for c in snap.columns if c.lower() not in snap.masked and snap.kind(c) == "number"] + return found[0] if len(found) == 1 else None + + +__all__ = ["CHECKS", "CHECK_NAMES", "Check", "Outcome", "STATUS_NAMES", "Span", "comparable", "compare_periods", + "compare_queries", "period_span", "reasons", "same_query", "span_coverage", "sql_span", "table_of", + "values_of"] diff --git a/backend/app/analysis/contract.py b/backend/app/analysis/contract.py index 8e82268d..75b7af52 100644 --- a/backend/app/analysis/contract.py +++ b/backend/app/analysis/contract.py @@ -208,6 +208,7 @@ class AnalysisInput(TypedDict, total=False): column: str value: str | None dialect: str # 读这份快照的 SQL 用的方言（第 1 波 W1-a 加，可选）：复算按它读，界的写法才一样 + masked: list[str] # 这份快照当时不能当维度、度量的列（小写，只记列名；第 1 波 W1-a 加，可选）：复算按它认 class Probe(TypedDict, total=False): diff --git a/backend/app/analysis/decompose.py b/backend/app/analysis/decompose.py new file mode 100644 index 00000000..f55224ce --- /dev/null +++ b/backend/app/analysis/decompose.py @@ -0,0 +1,732 @@ +"""变化拆解（细案 01 §2）：变化从哪来。三种模式，agent 按指标的性质选，算子核对选得对不对。 + +**参数**（AnalysisRequest.params；列名不分大小写，按快照认）： + +- 一份快照：query（编号）、period（期次列）、base / target（期次列里的两个值）；或两份快照：base_query、target_query（各自 + 只有维度列和度量列，先做可比性检查：口径结构不同的不拆，算不了 incomparable_sides；其余不可比的照样算，判定后附上 + 可比性结论）。 +- additive：by（维度列，1–2 列）、measure（度量列）；可选 headline {base, target}（基期、本期总数各一格，如 Q1.r1.gmv，用来 + 核对闭合）、parent {of: A3, key: [华东]}（下钻：这一层各段之和要等于 A3 里华东那一段）。 +- ratio：by、numerator（分子列）、denominator（分母列）；可选 headline {base, target}（整体比率的两格）、parent（上一层的比率 + 拆解）。 +- factor：product（乘积列）、factors（因子列，2–4 个）；一份快照按 period / base / target 取两行，两份快照各一行。 +- 都可以写 label（度量的叫法，消毒后进判定语）。 + +**计算**全程 Fraction（快照里的数按字面换成精确的有理数）：各段之和逐位等于总变化；比率拆解的结构效应、自身效应按中点 +对称拆，Σmix + Σrate 逐位等于比率的变化；乘积拆解是 LMDI-I（对数），浮点，各因子贡献之和在浮点误差内等于总变化。判定、 +阈值、边界照细案 §2.1–§2.4。 +""" +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from fractions import Fraction +from typing import Any + +from app.analysis import comparable as cmp +from app.analysis import phrases as ph +from app.analysis.common import Env, build, closes, column_label, label_for, need, refuse, sign_word, today_param +from app.analysis.contract import CallError +from app.analysis.inputs import Snap, as_list, hashable, same_value +from app.analysis.numbers import canon, to_fraction + +#: 辛普森：分母够大的段里，按平均权重算有这么多的段比率和整体反着变 +SIMPSON_SHARE = Fraction(4, 5) +#: 下钻、总数的写法（调用错误里教 agent 怎么写） +PARENT_SHAPE = "下钻要写成 {of: A3, key: [华东]} 这样：of 是上一层变化拆解的编号，key 是那一段的维度值" +HEADLINE_SHAPE = "总数要写成 {base: Q1.r1.gmv, target: Q1.r0.gmv} 这样：base 是基期总数那一格，target 是本期总数那一格" +SAME_QUERY = "基期和本期是同一份查询：两期各给一份查询，或者给一份按期次分组的查询和期次列" +#: 乘积闭合的最小相对误差 +_FACTOR_REL = Fraction(1, 10 ** 6) +_NOT_ADDITIVE = {"ratio": "比率", "identifier": "标识", "status": "状态", "attribute": "属性"} + + +@dataclass +class Side: + """一期的各段：键 → 值（ratio 模式是 (分子, 分母)）；nulls 是这一期度量格为空的段；rows 是用到的行。""" + + snap: Snap + values: dict[tuple[Any, ...], Any] = field(default_factory=dict) + nulls: list[tuple[Any, ...]] = field(default_factory=list) + rows: list[int] = field(default_factory=list) + + +@dataclass +class Segment: + key: tuple[Any, ...] + base: Any + target: Any + flags: list[str] = field(default_factory=list) + + +def _key_strings(key: Sequence[Any]) -> list[str]: + return [ph.key_text([k]) for k in key] + + +def _sides(raw: Mapping[str, Any], env: Env, roles: Sequence[tuple[str, str]], *, by_needed: bool = True + ) -> tuple[dict[str, Any], Snap, Snap, dict[str, str], list[str]]: + """一份或两份快照：(规范化的参数, 基期快照, 本期快照, {角色: 列名}, 维度列)。roles 是 [(参数名, 角色的叫法)]。""" + params: dict[str, Any] = {} + two = raw.get("base_query") is not None or raw.get("target_query") is not None + if two: + base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号") + target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号") + if base is target: + raise CallError("same_query", SAME_QUERY) + params.update(base_query=base.ref, target_query=target.ref) + else: + base = target = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号") + params["query"] = base.ref + by: list[str] = [] + if by_needed: + wanted = as_list(raw.get("by")) + if not 1 <= len(wanted) <= 2: + raise CallError("bad_by", "维度列要写 1 到 2 列") + by = [target.column(c, "维度列") for c in wanted] + if two: + for c in by: + base.column(c, "维度列") + params["by"] = by + cols: dict[str, str] = {} + for key, what in roles: + if key == "factors": + continue + col = target.column(need(raw, key, what), what) + if two: + base.column(col, what) + cols[key] = col + params[key] = col + if not two: + params["period"] = target.column(need(raw, "period", "期次列"), "期次列") + params["base"] = need(raw, "base", "基期（期次列里的值）") + params["target"] = need(raw, "target", "本期（期次列里的值）") + if same_value(params["base"], params["target"]): + raise CallError("same_period", "基期和本期是同一期：写期次列里两个不同的值") + used = [*cols.values(), *([params["period"]] if "period" in params else [])] + if len(set(by)) != len(by) or set(by) & set(used) or len(set(used)) != len(used): + raise CallError("bad_by", "维度列、期次列、度量列要是不同的列") + return params, base, target, cols, by + + +def _period_rows(snap: Snap, period: str, wanted: Any, what: str) -> list[int]: + i = snap.index(period) + rows = [r for r, row in enumerate(snap.rows) if i < len(row) and same_value(wanted, row[i])] + if not rows: + values = list(dict.fromkeys(ph.key_text([row[i]]) for row in snap.rows if i < len(row))) + raise CallError("period_not_found", f"期次列「{period}」里没有「{wanted}」：{what}要写 {snap.ref} 里有的期次值，" + f"比如 {'、'.join(values[:6])}") + return rows + + +def _rows_of(raw: Mapping[str, Any], params: dict[str, Any], base: Snap, target: Snap) -> tuple[list[int], list[int]]: + if "period" in params: + return (_period_rows(base, params["period"], params["base"], "基期"), + _period_rows(target, params["period"], params["target"], "本期")) + return list(range(len(base.rows))), list(range(len(target.rows))) + + +def _collect(snap: Snap, rows: Sequence[int], by: Sequence[str], read: Any, labels: Sequence[str], + period_label: str | None) -> Side: + """一期的各段：按维度键，同一键只许一行（多行是 SQL 没按这些维度汇总）。read(row) 读出这一行的值（None 是空）。""" + side = Side(snap) + seen: set[tuple[Any, ...]] = set() + idx = [snap.index(c) for c in by] + for r in rows: + row = snap.rows[r] + key = tuple(hashable(row[i] if i < len(row) else None) for i in idx) + if key in seen: + keys = "、".join([*labels, *([period_label] if period_label else [])]) + raise CallError("duplicate_rows", f"{snap.ref} 里同一个{'、'.join(labels)}在同一期有不止一行：在 SQL 里按 " + f"{keys} 汇总，每段每期一行") + seen.add(key) + value = read(r) + side.rows.append(r) + if value is None: + side.nulls.append(key) + side.values[key] = value + return side + + +def _segments(base: Side, target: Side, zero: Any) -> list[Segment]: + """两期的各段，按先出现的顺序：缺的一边记 0，标「新出现 / 消失」；度量格为空的记 0，标「为空」。""" + out: list[Segment] = [] + for key in dict.fromkeys([*base.values, *target.values]): + flags = [] + if key not in base.values: + flags.append("新出现") + if key not in target.values: + flags.append("消失") + if key in base.nulls or key in target.nulls: + flags.append("为空") + b, t = base.values.get(key), target.values.get(key) + out.append(Segment(key, zero if b is None else b, zero if t is None else t, flags)) + return out + + +def _parent(raw: Mapping[str, Any], env: Env, mode: str, dims: int) -> tuple[dict[str, Any], list[Any], int]: + """下钻：上一层的结论和那一段。返回 (params 里的 parent, 那一段在上一层结果表里的那一行, 这一层的层数)。""" + from app.core.artifact_store import canonical_json, content_hash + + spec = raw.get("parent") + name = spec.get("of") if isinstance(spec, Mapping) else None + key = as_list(spec.get("key")) if isinstance(spec, Mapping) else [] + if not isinstance(name, str) or not name.strip() or not key: + raise CallError("bad_parent", PARENT_SHAPE) + name = name.strip() + found = env.parents(name) if env.parents is not None else None + if not isinstance(found, Mapping): + raise CallError("parent_missing", f"没有叫「{name}」的分析结论：下钻写这次运行里变化拆解的编号（如 A3）") + if found.get("op") != "decompose" or found.get("mode") != mode: + raise CallError("parent_not_decompose", f"{name} 不是{'按维度' if mode == 'additive' else '比率'}的变化拆解，" + "下钻要接在同一种拆解后面") + if found.get("status") != "ok" or not isinstance(found.get("table"), Mapping): + raise CallError("parent_refused", f"{name} 没有做成，不能在它下面下钻") + depth = int((found.get("params") or {}).get("depth") or 0) + 1 + limit = int(env.settings["max_drill_depth"]) + if depth > limit: + raise CallError("too_deep", f"已经下钻了 {depth - 1} 层，最多 {limit} 层：换一个维度从头拆") + table = found["table"] + fixed = _FIXED_ADDITIVE if mode == "additive" else _FIXED_RATIO + columns = list(table.get("columns") or []) + dim_count = len([c for c in columns if c not in fixed]) + if len(key) != dim_count: + raise CallError("bad_parent", f"{name} 按 {dim_count} 列拆，下钻的那一段要写 {dim_count} 个维度值") + hit = next((row for row in table.get("rows") or [] + if all(same_value(k, v) for k, v in zip(key, row[:dim_count]))), None) + if hit is None: + raise CallError("parent_key_missing", f"{name} 里没有「{ph.key_text(key)}」这一段：写 {name} 结果表里的段") + env.inputs.parent(name, content_hash(canonical_json(found))) + return {"of": name, "key": key}, dict(zip(columns, hit)), depth + + +_FIXED_ADDITIVE = ("base", "target", "delta", "pct", "share", "flag") +_FIXED_RATIO = ("n_base", "n_target", "d_base", "d_target", "w_base", "w_target", "r_base", "r_target", "mix", "rate", + "total", "flag") + + +def _headline(raw: Mapping[str, Any], env: Env) -> tuple[dict[str, str] | None, dict[str, tuple[Fraction, Any]]]: + """headline：基期、本期总数各一格 → ({base, target} 的写法, {base / target: (值, 原样)})。""" + spec = raw.get("headline") + if spec is None: + return None, {} + if isinstance(spec, (list, tuple)) and len(spec) == 2: + spec = {"base": spec[0], "target": spec[1]} + if not isinstance(spec, Mapping) or not spec.get("base") or not spec.get("target"): + raise CallError("bad_headline", HEADLINE_SHAPE) + out: dict[str, tuple[Fraction, Any]] = {} + refs: dict[str, str] = {} + for side, what in (("base", "基期总数"), ("target", "本期总数")): + snap, row, col, value = env.inputs.cell(spec[side], f"headline_{side}", what) + if value is None: + raise CallError("headline_empty", f"{what}那一格（{snap.ref}.r{row}.{col}）是空的：换一格有值的") + out[side] = (value, snap.cell(row, col)) + refs[side] = f"{snap.ref}.r{row}.{col}" + return refs, out + + +def _measure_guard(env: Env, snap: Snap, column: str, caveats: list[dict[str, str]], tail: str) -> None: + """additive 的度量列：目录标成（确认过的）比率、标识、状态、属性是调用错误；没标的、只是推断的照算，写附带说明。""" + kind, confirmed = env.facts.measure(snap, column) + if kind in _NOT_ADDITIVE and confirmed: + if kind == "ratio": + raise CallError("ratio_as_additive", f"「{column}」是比率（或平均值），不能按段相加：改用比率拆解，分子、分母各给一列") + raise CallError("not_a_measure", f"「{column}」在目录里是{_NOT_ADDITIVE[kind]}，不是能相加的量：度量列要选金额、数量这类列") + if kind is None: + caveats.append(ph.caveat("measure_unlabeled", ph.unlabeled_text(column, tail))) + elif kind in _NOT_ADDITIVE: + caveats.append(ph.caveat("measure_unlabeled", ph.unconfirmed_text(column, kind, tail))) + + +def _comparability(env: Env, params: dict[str, Any], base: Snap, target: Snap, measure: str | None + ) -> tuple[cmp.Outcome | None, str | None]: + """两份快照时先做可比性：(结论, 口径结构不同的原因)。""" + if "base_query" not in params: + return None, None + params["today"] = today_param(env) + outcome = cmp.compare_queries(base, target, measure, env.facts, env.book, env.today) # type: ignore[arg-type] + return outcome, outcome.structure + + +def _share_rank(segments: Sequence[Any], share: Any) -> list[Any]: + """按 share 从大到小（一样大的按先出现的）。""" + return sorted(segments, key=lambda s: -share(s)) + + +# -------------------------------------------------------------------------- +# additive +# -------------------------------------------------------------------------- + + +def additive(raw: Mapping[str, Any], env: Env) -> Any: + used = ["main_share", "concentration", "offset_ratio", "flat_pct", "table_top_k"] + params, bs, ts, cols, by = _sides(raw, env, [("measure", "度量列")]) + measure = cols["measure"] + label = label_for(env, raw.get("label"), ts, measure) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + dims = [column_label(env, ts, c) for c in by] + caveats: list[dict[str, str]] = [] + _measure_guard(env, ts, measure, caveats, "按可以按段相加处理") + refs, heads = _headline(raw, env) + if refs: + params["headline"] = refs + parent_row = None + if raw.get("parent") is not None: + params["parent"], parent_row, params["depth"] = _parent(raw, env, "additive", len(by)) + used.append("max_drill_depth") + summed = not heads and parent_row is None + criterion_args = {"summed": summed} + for snap in dict.fromkeys([bs, ts]): + if snap.truncated: + env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [*by, measure]) + return refuse(env, params, used, "truncated", + f"{snap.ref} 只取回了前 {len(snap.rows)} 行，各段之和不是完整的和", + criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args)) + rows_b, rows_t = _rows_of(raw, params, bs, ts) + period_label = column_label(env, ts, params["period"]) if "period" in params else None + side_b = _collect(bs, rows_b, by, lambda r: bs.number(r, measure, "度量列"), dims, period_label) + side_t = _collect(ts, rows_t, by, lambda r: ts.number(r, measure, "度量列"), dims, period_label) + if bs is ts: + env.inputs.use(ts, "data", [*by, params["period"], measure], side_b.rows + side_t.rows) + else: + env.inputs.use(bs, "base", [*by, measure], side_b.rows) + env.inputs.use(ts, "target", [*by, measure], side_t.rows) + outcome, structure = _comparability(env, params, bs, ts, measure) + if structure is not None: + return refuse(env, params, used, "incomparable_sides", f"{structure}，不是同一个量的两期，不拆", + criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats) + segs = _segments(side_b, side_t, Fraction(0)) + if len(segs) < 2: + return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(segs)} 段，至少要 2 段才拆得开", + criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats) + return _additive_result(env, params, used, segs, by, dims, label, heads, parent_row, outcome, caveats, summed, + side_b, side_t) + + +def _additive_result(env: Env, params: dict[str, Any], used: list[str], segs: list[Segment], by: list[str], + dims: list[str], label: str, heads: Mapping[str, tuple[Fraction, Any]], + parent_row: Mapping[str, Any] | None, outcome: cmp.Outcome | None, + caveats: list[dict[str, str]], summed: bool, side_b: Side, side_t: Side) -> Any: + s = env.settings + main_share, concentration = env.threshold("main_share"), env.threshold("concentration") + offset_ratio, flat_pct = env.threshold("offset_ratio"), env.threshold("flat_pct") + B = sum((x.base for x in segs), Fraction(0)) + T = sum((x.target for x in segs), Fraction(0)) + delta = T - B + gaps: dict[str, Fraction] = {} + for side, total in (("base", B), ("target", T)): + if side in heads and not closes(total, heads[side][0], heads[side][1]): + gaps[side] = heads[side][0] - total + if parent_row is not None: + expect = to_fraction(parent_row.get(side)) + if expect is None or not closes(total, expect, parent_row.get(side)): + gaps.setdefault(side, (expect or Fraction(0)) - total) + closed = not gaps + deltas = {x.key: x.target - x.base for x in segs} + shares = {k: d / delta for k, d in deltas.items()} if closed and delta != 0 else {} + gross_up = sum((d for d in deltas.values() if d > 0), Fraction(0)) + gross_down = sum((d for d in deltas.values() if d < 0), Fraction(0)) + opposite = abs(gross_down) if delta > 0 else gross_up if delta < 0 else gross_up + n = len(segs) + k_max = max(2, math.ceil(n / 5)) + flat = delta == 0 or (B != 0 and abs(delta) / abs(B) < flat_pct) + ranked = _share_rank(segs, lambda x: shares[x.key]) if shares else [] + top_k = top_k_share = None + if ranked: + running = Fraction(0) + for k, seg in enumerate(ranked, start=1): + running += shares[seg.key] + if running >= concentration: + top_k, top_k_share = k, running + break + if not closed: + code, subjects = "unclosed", [] + elif flat: + code, subjects = "flat", [] + elif shares[ranked[0].key] >= main_share: + code, subjects = "driver", [ranked[0]] + elif top_k is not None and 2 <= top_k <= k_max: + code, subjects = "concentrated", ranked[:top_k] + else: + code, subjects = "dispersed", [] + offset = opposite > 0 and opposite >= offset_ratio * abs(delta) + if offset: + caveats.append(ph.caveat("offset", ph.offset_text(delta, gross_up, gross_down))) + nulls = [x for x in segs if "为空" in x.flags] + if nulls: + caveats.append(ph.caveat("null_values", ph.null_segments_text([ph.key_text(x.key) for x in nulls], label))) + if summed: + caveats.append(ph.caveat("headline_basis", ph.HEADLINE_BASIS)) + if outcome is not None and outcome.verdict != "comparable": + caveats.append(ph.caveat("comparability", outcome.phrase)) + top = ranked[0] if ranked else max(segs, key=lambda x: abs(deltas[x.key])) + total_text = ph.change(delta, B) + top_n = int(s["table_top_k"]) + if code == "driver": + main = ph.decompose_driver(label, total_text, ph.key_text(top.key), ph.amount(deltas[top.key], signed=True), + shares[top.key]) + elif code == "concentrated": + main = ph.decompose_concentrated(label, total_text, ph.names([ph.key_text(x.key) for x in subjects], top_n), + top_k_share) + elif code == "dispersed": + main = ph.decompose_dispersed(label, total_text, ph.dims_text(dims), main_share) + elif code == "flat": + ups = [x for x in segs if deltas[x.key] > 0] + downs = [x for x in segs if deltas[x.key] < 0] + pair = None + if offset and ups and downs: + up = max(ups, key=lambda x: deltas[x.key]) + down = min(downs, key=lambda x: deltas[x.key]) + pair = (ph.key_text(up.key), ph.amount(deltas[up.key], signed=True), ph.key_text(down.key), + ph.amount(deltas[down.key], signed=True)) + main = ph.decompose_flat(label, delta / abs(B) if B != 0 else None, pair) + else: + gap = max(gaps.values(), key=abs) + main = ph.decompose_unclosed(ph.dims_text(dims), gap, ratio=False) + order = sorted(segs, key=lambda x: -abs(deltas[x.key])) + rows = [] + for x in order: + d = deltas[x.key] + rows.append([*x.key, canon(x.base), canon(x.target), canon(d), canon(d / x.base) if x.base != 0 else None, + canon(shares[x.key]) if shares else None, "、".join(x.flags) or None]) + table = {"columns": [*by, *_FIXED_ADDITIVE], + "labels": {**dict(zip(by, dims)), "base": "基期", "target": "本期", "delta": "变化", "pct": "变化率", + "share": "占变化的比重", "flag": "说明"}, + "rows": rows} + values: dict[str, Any] = { + "base_total": canon(B), "target_total": canon(T), "delta_total": canon(delta), + "pct_total": canon(delta / abs(B)) if B != 0 else None, "top_key": ph.key_text(top.key), + "top_share": canon(shares[top.key]) if shares else None, "top_k": str(top_k) if top_k is not None else None, + "top_k_share": canon(top_k_share) if top_k_share is not None else None, "gross_up": canon(gross_up), + "gross_down": canon(gross_down), "segments": str(n), + } + if gaps: + values["gap"] = canon(max(gaps.values(), key=abs)) + verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects], + "direction": sign_word(delta) if code in ("driver", "concentrated", "dispersed") else None} + phrase = ph.with_caveats(main, caveats, skip=("offset",) if code == "flat" else ()) + return build(env, params, used, table=table, values=values, verdict=verdict, + criterion=ph.criterion_additive(s, top_k_max=k_max, summed=summed), phrase=phrase, caveats=caveats) + + +# -------------------------------------------------------------------------- +# ratio +# -------------------------------------------------------------------------- + + +def ratio(raw: Mapping[str, Any], env: Env) -> Any: + used = ["main_share", "flat_pct", "min_denominator", "table_top_k"] + params, bs, ts, cols, by = _sides(raw, env, [("numerator", "分子列"), ("denominator", "分母列")]) + num, den = cols["numerator"], cols["denominator"] + num_label, den_label = column_label(env, ts, num), column_label(env, ts, den) + label = ph.pick_label(raw.get("label"), ph.say(num_label, "与", den_label, "之比")) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + dims = [column_label(env, ts, c) for c in by] + refs, heads = _headline(raw, env) + if refs: + params["headline"] = refs + parent_row = None + if raw.get("parent") is not None: + params["parent"], parent_row, params["depth"] = _parent(raw, env, "ratio", len(by)) + used.append("max_drill_depth") + criterion = ph.criterion_ratio(env.settings) + for snap in dict.fromkeys([bs, ts]): + if snap.truncated: + env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [*by, num, den]) + return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，各段合起来不是完整的数", + criterion=criterion) + rows_b, rows_t = _rows_of(raw, params, bs, ts) + period_label = column_label(env, ts, params["period"]) if "period" in params else None + + def reader(snap: Snap) -> Any: + return lambda r: (snap.number(r, num, "分子列"), snap.number(r, den, "分母列")) + + side_b = _collect(bs, rows_b, by, reader(bs), dims, period_label) + side_t = _collect(ts, rows_t, by, reader(ts), dims, period_label) + for side in (side_b, side_t): + side.nulls = [k for k, v in side.values.items() if v[0] is None or v[1] is None] + side.values = {k: (v[0] or Fraction(0), v[1] or Fraction(0)) for k, v in side.values.items()} + if bs is ts: + env.inputs.use(ts, "data", [*by, params["period"], num, den], side_b.rows + side_t.rows) + else: + env.inputs.use(bs, "base", [*by, num, den], side_b.rows) + env.inputs.use(ts, "target", [*by, num, den], side_t.rows) + outcome, structure = _comparability(env, params, bs, ts, den) + if structure is not None: + return refuse(env, params, used, "incomparable_sides", f"{structure}，不是同一个量的两期，不拆", criterion=criterion) + segs = _segments(side_b, side_t, (Fraction(0), Fraction(0))) + live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0] + if len(live) < 2: + return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(live)} 段，至少要 2 段才拆得开", + criterion=criterion) + for when, i in (("基期", 0), ("本期", 1)): + if sum(((x.base, x.target)[i][1] for x in live), Fraction(0)) == 0: + return refuse(env, params, used, "zero_base", ph.say(f"{when}的", den_label, "合计为 0，算不了比率"), + criterion=criterion) + for x in live: + for (n_, d_), when in ((x.base, "基期"), (x.target, "本期")): + if d_ == 0 and n_ != 0: + return refuse(env, params, used, "zero_base", + ph.say(ph.key_text(x.key), f"{when}的", den_label, "为 0 而", num_label, + "不为 0，比率拆不了"), criterion=criterion) + return _ratio_result(env, params, used, segs, by, dims, label, num_label, den_label, heads, parent_row, outcome, + criterion) + + +def ratio_effects(segs: Sequence[tuple[Fraction, Fraction, Fraction, Fraction]] + ) -> tuple[Fraction, Fraction, list[tuple[Fraction, ...]]]: + """比率拆解的计算（细案 §2.2，中点对称拆法）。segs 是各段 (分子_基期, 分母_基期, 分子_本期, 分母_本期)，两期分母都为 0 + 的段先剔掉。交回 (R, R', 各段 (w, w', r, r', mix, rate))：Σmix + Σrate = R' − R 逐位成立（Σ(w' − w) = 0）。一边分母为 0 + 的段：那一边权重 0、比率取另一边的（自身效应为 0，变化全记在结构上）。""" + db = sum((s[1] for s in segs), Fraction(0)) + dt = sum((s[3] for s in segs), Fraction(0)) + R = sum((s[0] for s in segs), Fraction(0)) / db + R2 = sum((s[2] for s in segs), Fraction(0)) / dt + mid = (R + R2) / 2 + out = [] + for nb, d_b, nt, d_t in segs: + w, w2 = d_b / db, d_t / dt + r = nb / d_b if d_b != 0 else None + r2 = nt / d_t if d_t != 0 else None + r = r2 if r is None else r + r2 = r if r2 is None else r2 + mix = (w2 - w) * ((r + r2) / 2 - mid) # type: ignore[operator] + rate = (r2 - r) * (w + w2) / 2 # type: ignore[operator] + out.append((w, w2, r, r2, mix, rate)) + return R, R2, out + + +def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[Segment], by: list[str], + dims: list[str], label: str, num_label: str, den_label: str, + heads: Mapping[str, tuple[Fraction, Any]], parent_row: Mapping[str, Any] | None, + outcome: cmp.Outcome | None, criterion: str) -> Any: + main_share, flat_pct = env.threshold("main_share"), env.threshold("flat_pct") + min_den = env.threshold("min_denominator") + live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0] + dropped = [x for x in segs if x not in live] + R, R2, effects = ratio_effects([(x.base[0], x.base[1], x.target[0], x.target[1]) for x in live]) + delta = R2 - R + by_key = {x.key: e for x, e in zip(live, effects)} + gaps: dict[str, Fraction] = {} + for side, total in (("base", R), ("target", R2)): + if side in heads and not closes(total, heads[side][0], heads[side][1]): + gaps[side] = heads[side][0] - total + if parent_row is not None: + sums = {"n_base": sum((x.base[0] for x in live), Fraction(0)), "d_base": sum((x.base[1] for x in live), Fraction(0)), + "n_target": sum((x.target[0] for x in live), Fraction(0)), + "d_target": sum((x.target[1] for x in live), Fraction(0))} + for col, total in sums.items(): + expect = to_fraction(parent_row.get(col)) + if expect is None or not closes(total, expect, parent_row.get(col)): + gaps.setdefault("base" if col.endswith("base") else "target", (expect or Fraction(0)) - total) + closed = not gaps + mix_total = sum((e[4] for e in effects), Fraction(0)) + rate_total = sum((e[5] for e in effects), Fraction(0)) + flat = delta == 0 or (R != 0 and abs(delta) / abs(R) < flat_pct) + mix_share = mix_total / delta if delta != 0 else None + rate_share = rate_total / delta if delta != 0 else None + sign = 1 if delta >= 0 else -1 + if not closed: + code, subjects = "unclosed", [] + elif flat: + code, subjects = "flat", [] + elif mix_share >= main_share and rate_share < main_share: # type: ignore[operator] + code = "mix_driven" + subjects = [max(live, key=lambda x: sign * by_key[x.key][4])] + elif rate_share >= main_share and mix_share < main_share: # type: ignore[operator] + code = "rate_driven" + subjects = [max(live, key=lambda x: sign * by_key[x.key][5])] + else: + code, subjects = "both", [] + proportion = all(0 <= v[0] <= v[1] for x in live for v in (x.base, x.target)) + small = [x for x in live if x.base[1] < min_den or x.target[1] < min_den] + caveats: list[dict[str, str]] = [] + eligible = [x for x in live if x not in small] + if delta != 0 and eligible: + weight = sum(((by_key[x.key][0] + by_key[x.key][1]) / 2 for x in eligible), Fraction(0)) + against = sum(((by_key[x.key][0] + by_key[x.key][1]) / 2 for x in eligible + if (by_key[x.key][3] - by_key[x.key][2]) * sign < 0), Fraction(0)) + if weight > 0 and against / weight >= SIMPSON_SHARE: + star = max(live, key=lambda x: sign * by_key[x.key][4]) + caveats.append(ph.caveat("simpson", ph.simpson_text(label, delta > 0, ph.key_text(star.key), + by_key[star.key][0], by_key[star.key][1]))) + if small: + caveats.append(ph.caveat("small_denominator", ph.small_denominator_text( + [ph.key_text(x.key) for x in small], den_label, int(env.settings["min_denominator"])))) + nulls = [x for x in segs if "为空" in x.flags] + if nulls: + caveats.append(ph.caveat("null_values", ph.null_segments_text([ph.key_text(x.key) for x in nulls], + ph.say(num_label, "或", den_label)))) + if outcome is not None and outcome.verdict != "comparable": + caveats.append(ph.caveat("comparability", outcome.phrase)) + total_text = ph.ratio_change(R, R2, proportion=proportion) + if code == "mix_driven": + star = subjects[0] + main = ph.ratio_mix(label, total_text, ph.key_text(star.key), by_key[star.key][0], by_key[star.key][1]) + elif code == "rate_driven": + main = ph.ratio_rate(label, total_text, ph.key_text(subjects[0].key)) + elif code == "both": + main = ph.ratio_both(label, total_text, mix_share, rate_share) + elif code == "flat": + main = ph.decompose_flat(label, delta / abs(R) if R != 0 else None, None) + else: + main = ph.decompose_unclosed(ph.dims_text(dims), max(gaps.values(), key=abs), ratio=True) + rows = [] + for x in sorted(live, key=lambda x: -abs(by_key[x.key][4] + by_key[x.key][5])): + w, w2, r, r2, mix, rate = by_key[x.key] + flags = list(x.flags) + if x.base[1] == 0: + flags.append("基期分母为 0") + if x.target[1] == 0: + flags.append("本期分母为 0") + if x in small: + flags.append("分母小") + rows.append([*x.key, canon(x.base[0]), canon(x.target[0]), canon(x.base[1]), canon(x.target[1]), canon(w), + canon(w2), canon(r), canon(r2), canon(mix), canon(rate), canon(mix + rate), + "、".join(dict.fromkeys(flags)) or None]) + for x in dropped: + rows.append([*x.key, canon(x.base[0]), canon(x.target[0]), "0", "0", None, None, None, None, None, None, None, + "两期分母都为 0，不计入"]) + table = {"columns": [*by, *_FIXED_RATIO], + "labels": {**dict(zip(by, dims)), "n_base": ph.say("基期", num_label), "n_target": ph.say("本期", num_label), + "d_base": ph.say("基期", den_label), "d_target": ph.say("本期", den_label), + "w_base": "基期权重", "w_target": "本期权重", "r_base": "基期比率", "r_target": "本期比率", + "mix": "结构效应", "rate": "自身效应", "total": "合计影响", "flag": "说明"}, + "rows": rows} + values: dict[str, Any] = { + "r_base_total": canon(R), "r_target_total": canon(R2), "delta": canon(delta), "mix_total": canon(mix_total), + "rate_total": canon(rate_total), "mix_share": canon(mix_share), "rate_share": canon(rate_share), + "segments": str(len(live)), + } + if gaps: + values["gap"] = canon(max(gaps.values(), key=abs)) + verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects], + "direction": sign_word(delta) if code in ("mix_driven", "rate_driven", "both") else None} + return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, + phrase=ph.with_caveats(main, caveats), caveats=caveats) + + +# -------------------------------------------------------------------------- +# factor +# -------------------------------------------------------------------------- + + +def _half_ulp_ratio(raw: Any, value: Fraction) -> Fraction: + from app.analysis.common import written_places + + return Fraction(1, 2) / Fraction(10) ** written_places(raw) / abs(value) if value != 0 else Fraction(0) + + +def lmdi(base: Sequence[float], target: Sequence[float], m_base: float, m_target: float) -> list[float]: + """LMDI-I：contrib_k = L(M', M) · ln(F_k' / F_k)，L(a, b) = (a − b) / (ln a − ln b)，L(a, a) = a。因子按书写精度取过整、 + 乘积和乘积列差一点时，各因子按同一比例缩放到乘积恰好等于乘积列（每个因子的对数比各补上 (1/K)·差的对数），Σcontrib 在 + 浮点误差内等于 M' − M。""" + k = len(base) + fix = (math.log(m_target / math.prod(target)) - math.log(m_base / math.prod(base))) / k + weight = m_base if m_base == m_target else (m_target - m_base) / (math.log(m_target) - math.log(m_base)) + return [weight * (math.log(t / b) + fix) for b, t in zip(base, target)] + + +def factor(raw: Mapping[str, Any], env: Env) -> Any: + used = ["main_share", "flat_pct"] + params, bs, ts, cols, _ = _sides(raw, env, [("product", "乘积列")], by_needed=False) + product = cols["product"] + wanted = as_list(raw.get("factors")) + if not 2 <= len(wanted) <= 4: + raise CallError("bad_factors", "因子列要写 2 到 4 列") + factors = [ts.column(c, "因子列") for c in wanted] + if bs is not ts: + for c in factors: + bs.column(c, "因子列") + if len(set(factors)) != len(factors) or product in factors: + raise CallError("bad_factors", "因子列不能重复，也不能是乘积列本身") + params["factors"] = factors + label = label_for(env, raw.get("label"), ts, product) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + names = [column_label(env, ts, c) for c in factors] + criterion = ph.criterion_factor(env.settings) + for snap in dict.fromkeys([bs, ts]): + if snap.truncated: + env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [product, *factors]) + return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，结果不完整", + criterion=criterion) + if "period" in params: + rows_b = _period_rows(bs, params["period"], params["base"], "基期") + rows_t = _period_rows(ts, params["period"], params["target"], "本期") + else: + rows_b, rows_t = list(range(len(bs.rows))), list(range(len(ts.rows))) + for rows, snap, when in ((rows_b, bs, "基期"), (rows_t, ts, "本期")): + if len(rows) != 1: + raise CallError("duplicate_rows", f"{snap.ref} 里{when}有 {len(rows)} 行：乘积拆解每期要正好一行（在 SQL 里汇总成一行）") + cells: dict[str, list[tuple[Fraction, Any]]] = {} + for when, snap, r in (("base", bs, rows_b[0]), ("target", ts, rows_t[0])): + row = [] + for col, what in ((product, "乘积列"), *((c, "因子列") for c in factors)): + value = snap.number(r, col, what) + if value is None: + raise CallError("empty_cell", f"{snap.ref} 第 {r} 行的「{col}」是空的：乘积、因子都要有值") + row.append((value, snap.cell(r, col))) + cells[when] = row + if bs is ts: + env.inputs.use(ts, "data", [params["period"], product, *factors], rows_b + rows_t) + else: + env.inputs.use(bs, "base", [product, *factors], rows_b) + env.inputs.use(ts, "target", [product, *factors], rows_t) + outcome, structure = _comparability(env, params, bs, ts, product) + if structure is not None: + return refuse(env, params, used, "incomparable_sides", f"{structure}，不是同一个量的两期，不拆", criterion=criterion) + if any(v <= 0 for row in cells.values() for v, _ in row): + return refuse(env, params, used, "non_positive", "乘积或因子里有不大于 0 的值，对数拆解要求全为正", criterion=criterion) + for when, row in cells.items(): + m = row[0][0] + prod = math.prod((v for v, _ in row[1:]), start=Fraction(1)) + rel = abs(m - prod) / abs(m) + allowed = max(_FACTOR_REL, sum((_half_ulp_ratio(raw_v, v) for v, raw_v in row[1:]), Fraction(0))) + if rel > allowed: + raise CallError("not_product", ph.say("这几列相乘不等于", column_label(env, ts, product), + f"（{'基期' if when == 'base' else '本期'}差 {ph.pct(rel, signed=False)}），" + "不是乘积关系：换成乘起来正好等于它的几列")) + M, M2 = cells["base"][0][0], cells["target"][0][0] + fb = [float(v) for v, _ in cells["base"][1:]] + ft = [float(v) for v, _ in cells["target"][1:]] + contrib = lmdi(fb, ft, float(M), float(M2)) + delta = M2 - M + flat = delta == 0 or abs(delta) / abs(M) < env.threshold("flat_pct") + shares = [c / float(delta) for c in contrib] if delta != 0 else [None] * len(contrib) + main_share = float(env.threshold("main_share")) + order = sorted(range(len(factors)), key=lambda i: -abs(contrib[i])) + caveats: list[dict[str, str]] = [] + if outcome is not None and outcome.verdict != "comparable": + caveats.append(ph.caveat("comparability", outcome.phrase)) + total_text = ph.change(delta, M) + top = max(range(len(factors)), key=lambda i: shares[i] if shares[i] is not None else -math.inf) + if flat: + code, subjects = "flat", [] + main = ph.decompose_flat(label, delta / abs(M), None) + elif shares[top] >= main_share: # type: ignore[operator] + code, subjects = "driver", [[names[top]]] + main = ph.factor_driver(label, total_text, names[top], contrib[top], shares[top]) + else: + code, subjects = "both", [] + main = ph.factor_both(label, total_text, [(names[i], shares[i]) for i in order]) + rows = [] + for i in order: + b, t = cells["base"][i + 1][0], cells["target"][i + 1][0] + rows.append([names[i], canon(b), canon(t), canon((t - b) / b), canon(contrib[i]), + canon(shares[i]) if shares[i] is not None else None]) + table = {"columns": ["factor", "base", "target", "pct", "contrib", "share"], + "labels": {"factor": "因子", "base": "基期", "target": "本期", "pct": "变化率", "contrib": "贡献", + "share": "占变化的比重"}, + "rows": rows} + values = {"base_total": canon(M), "target_total": canon(M2), "delta_total": canon(delta), + "pct_total": canon(delta / abs(M)), "top_key": names[top], + "top_share": canon(shares[top]) if shares[top] is not None else None} + verdict = {"code": code, "subjects": subjects, "direction": sign_word(delta) if code != "flat" else None} + return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, + phrase=ph.with_caveats(main, caveats), caveats=caveats) + + +__all__ = ["HEADLINE_SHAPE", "PARENT_SHAPE", "SAME_QUERY", "SIMPSON_SHARE", "additive", "factor", "lmdi", "ratio", "ratio_effects"] diff --git a/backend/app/analysis/differ.py b/backend/app/analysis/differ.py new file mode 100644 index 00000000..a6d5e7ef --- /dev/null +++ b/backend/app/analysis/differ.py @@ -0,0 +1,372 @@ +"""差异检验（细案 01 §5）：这两组（或两期）真的不一样，还是样本波动。 + +**参数**：query（一份快照）；按组比时 by（分组列，一列）；按期比同一组时 period、base、target（各一行，SQL 里先筛好那一组）。 +比哪几对：pairs（[[A, B], …]，A、B 是分组列里的值）、reference（对照组：其余各组各和它比）；都不写时两组就比这两组，三组 +以上每组和「其余各组合计」比（和含它自己的整体比不独立）。可选 label。 + +- rate：numerator（成功数，如转化单数）、denominator（试验数，如访客数），每组一行，都要是不小于 0 的整数、成功数不超过 + 试验数。每组 Wilson 区间，两组之差 Newcombe 区间；最小期望频数不小于 min_expected 用合并比例的 z 检验，否则用 Fisher + 精确检验。 +- mean：每组一行，mean、sd、n 三列（均值、标准差、样本数），或者 sum、sumsq、n 三列（合计、平方和、样本数：SQLite 没有标准差 + 函数，SUM(x)、SUM(x*x)、COUNT(*) 都写得出）。Welch t 检验，自由度按 Welch–Satterthwaite。 +- rows：value（逐行的值）+ 分组列（或期次列），快照不超过 1000 行、没截断；算子自己算均值、方差，另做敏感性：去掉离均值 + 最远的那一个值后重算，判定变了加 outlier_sensitive。 + +比了几对时按 Holm 校正，表里同时给原始 p 和校正后的 p，附 multiple_comparisons。校正后 p < differ_alpha → differs（direction +按 A − B 的正负）；否则 no_evidence。用了 Fisher（比率）或有一组样本数小于 min_n（均值）→ small_sample。判定语不出现「p 值」 +「显著性」：differs 说「差异站得住」，no_evidence 说「差异可能只是样本波动」，p 值、区间在表和证据面板里。 + +subjects 是判定说的那一对 [[A], [B]]（只比一对时就是那一对，比了几对时是校正后最站得住的那一对；几对都不站得住时为空）， +direction 是 A 相对 B。 +""" +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from fractions import Fraction +from typing import Any + +from app.analysis import phrases as ph +from app.analysis import stats +from app.analysis.common import Env, build, column_label, label_for, need, refuse, sign_word +from app.analysis.contract import CallError +from app.analysis.inputs import Snap, as_list, hashable, same_value +from app.analysis.numbers import canon, to_fraction + +TESTS = {"z": "z 检验", "fisher": "Fisher 精确检验", "welch": "Welch t 检验"} + + +@dataclass(eq=False) +class Group: + key: Any + name: str + rows: list[int] = field(default_factory=list) + x: int = 0 # rate：成功数 + n: int = 0 # rate：试验数；mean / rows：样本数 + mean: Fraction | None = None + var: Fraction | None = None + values: list[Fraction] = field(default_factory=list) + + +@dataclass +class Pair: + a: Group + b: Group + test: str | None = None + p: float | None = None + p_adj: float | None = None + diff: Fraction | None = None + ci: tuple[float, float] | None = None + a_value: Fraction | None = None + b_value: Fraction | None = None + + +def _int(value: Fraction | None, snap: Snap, r: int, column: str) -> int: + if value is None or value.denominator != 1 or value < 0: + raise CallError("not_count", f"{snap.ref} 第 {r} 行的「{column}」不是不小于 0 的整数：比率比较的分子、分母要是计数") + return int(value) + + +def _pool(name: str, groups: Sequence[Group], mode: str) -> Group: + """「其余各组合计」：比率把成功数、试验数加起来；均值按各组的均值、方差、样本数合并；逐行的值并在一起。""" + out = Group(("__rest__",), name, rows=[r for g in groups for r in g.rows]) + if mode == "rate": + out.x, out.n = sum(g.x for g in groups), sum(g.n for g in groups) + return out + if mode == "rows": + out.values = [v for g in groups for v in g.values] + return _moments(out) + out.n = sum(g.n for g in groups) + if out.n == 0: + return out + out.mean = sum((g.mean * g.n for g in groups if g.mean is not None), Fraction(0)) / out.n + if out.n > 1: + within = sum(((g.n - 1) * g.var for g in groups if g.var is not None and g.n > 1), Fraction(0)) + between = sum((g.n * (g.mean - out.mean) ** 2 for g in groups if g.mean is not None), Fraction(0)) + out.var = (within + between) / (out.n - 1) + return out + + +def _moments(group: Group) -> Group: + group.n = len(group.values) + if group.n: + group.mean = sum(group.values, Fraction(0)) / group.n + if group.n > 1: + group.var = sum(((v - group.mean) ** 2 for v in group.values), Fraction(0)) / (group.n - 1) # type: ignore[operator] + return group + + +def _test(pair: Pair, mode: str, min_expected: Fraction) -> None: + a, b = pair.a, pair.b + if mode == "rate": + if a.n == 0 or b.n == 0: + return + pair.a_value, pair.b_value = Fraction(a.x, a.n), Fraction(b.x, b.n) + pair.diff = pair.a_value - pair.b_value + pair.ci = stats.newcombe(a.x, a.n, b.x, b.n) + if stats.min_expected(a.x, a.n, b.x, b.n) >= min_expected: + pair.test = "z" + pair.p = stats.pooled_z(a.x, a.n, b.x, b.n)[1] # type: ignore[index] + else: + pair.test = "fisher" + pair.p = stats.fisher_exact(a.x, a.n - a.x, b.x, b.n - b.x) + return + if a.mean is None or b.mean is None: + return + pair.a_value, pair.b_value = a.mean, b.mean + pair.diff = a.mean - b.mean + if a.var is None or b.var is None: + return + test = stats.welch(float(a.mean), float(a.var), a.n, float(b.mean), float(b.var), b.n) + if test is None: + return + pair.test, pair.p = "welch", test.p + pair.ci = stats.welch_interval(float(a.mean), float(a.var), a.n, float(b.mean), float(b.var), b.n) + + +def _verdict(pairs: Sequence[Pair], alpha: Fraction) -> list[Pair]: + """Holm 校正后站得住的那几对（按校正后的 p 从小到大，一样的按先后）。""" + adjusted = stats.holm([p.p for p in pairs]) + for pair, adj in zip(pairs, adjusted): + pair.p_adj = adj + hits = [p for p in pairs if p.p_adj is not None and to_fraction(p.p_adj) < alpha] # type: ignore[operator] + return sorted(hits, key=lambda p: p.p_adj) # type: ignore[arg-type, return-value] + + +def differ(raw: Mapping[str, Any], env: Env) -> Any: + mode = env.mode + used = ["differ_alpha", "min_expected"] if mode == "rate" else ["differ_alpha", "min_n"] + snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号") + params: dict[str, Any] = {"query": snap.ref} + by_period = raw.get("period") is not None + if by_period: + group_col = snap.column(raw.get("period"), "期次列") + params.update(period=group_col, base=need(raw, "base", "基期（期次列里的值）"), + target=need(raw, "target", "本期（期次列里的值）")) + else: + wanted = as_list(raw.get("by")) + if len(wanted) != 1: + raise CallError("bad_by", "差异检验按一列分组：分组列写一列；比同一组的两期就写期次列、基期、本期") + group_col = snap.column(wanted[0], "分组列") + params["by"] = [group_col] + if mode == "rate": + cols = {"numerator": snap.column(need(raw, "numerator", "成功数列"), "成功数列"), + "denominator": snap.column(need(raw, "denominator", "试验数列"), "试验数列")} + elif mode == "mean": + if raw.get("mean") is not None: + cols = {"mean": snap.column(raw.get("mean"), "均值列"), "sd": snap.column(need(raw, "sd", "标准差列"), "标准差列"), + "n": snap.column(need(raw, "n", "样本数列"), "样本数列")} + else: + cols = {"sum": snap.column(need(raw, "sum", "合计列（或均值列）"), "合计列"), + "sumsq": snap.column(need(raw, "sumsq", "平方和列"), "平方和列"), + "n": snap.column(need(raw, "n", "样本数列"), "样本数列")} + else: + cols = {"value": snap.column(need(raw, "value", "值列"), "值列")} + params.update(cols) + measure_col = cols.get("numerator") or cols.get("mean") or cols.get("sum") or cols.get("value") + if mode == "rate": + label = ph.pick_label(raw.get("label"), ph.say(column_label(env, snap, cols["numerator"]), "占", + column_label(env, snap, cols["denominator"]), "的比率")) + else: + label = label_for(env, raw.get("label"), snap, measure_col) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + dim_label = column_label(env, snap, group_col) + criterion = ph.criterion_differ(env.settings, mode) # type: ignore[arg-type] + if snap.truncated: + env.inputs.use(snap, "data", [group_col, *cols.values()]) + return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，各组不完整", + criterion=criterion) + groups = _groups(snap, group_col, cols, mode, by_period, params) # type: ignore[arg-type] + pairs = _pairs(raw, params, groups, mode, dim_label, by_period, group_col) # type: ignore[arg-type] + env.inputs.use(snap, "data", [group_col, *cols.values()], + sorted({r for p in pairs for g in (p.a, p.b) for r in g.rows})) + return _decide(env, params, used, pairs, mode, label, criterion) # type: ignore[arg-type] + + +def _groups(snap: Snap, group_col: str, cols: Mapping[str, str], mode: str, by_period: bool, + params: Mapping[str, Any]) -> dict[Any, Group]: + gi = snap.index(group_col) + groups: dict[Any, Group] = {} + wanted = [params["target"], params["base"]] if by_period else None + for r, row in enumerate(snap.rows): + value = row[gi] if gi < len(row) else None + if wanted is not None: + if not any(same_value(w, value) for w in wanted): + continue + key = hashable(value) + group = groups.setdefault(key, Group(key, ph.key_text([value]))) + if mode != "rows" and group.rows: + raise CallError("duplicate_rows", f"{snap.ref} 里「{ph.key_text([value])}」有不止一行：在 SQL 里按 {group_col} " + "汇总，每组一行") + group.rows.append(r) + if mode == "rate": + group.x = _int(snap.number(r, cols["numerator"], "成功数列"), snap, r, cols["numerator"]) + group.n = _int(snap.number(r, cols["denominator"], "试验数列"), snap, r, cols["denominator"]) + if group.x > group.n: + raise CallError("not_count", f"{snap.ref} 第 {r} 行的成功数大于试验数：分子、分母要是同一批里的「成功几个、" + "一共几个」") + elif mode == "mean": + n = _int(snap.number(r, cols["n"], "样本数列"), snap, r, cols["n"]) + group.n = n + if "mean" in cols: + group.mean = snap.number(r, cols["mean"], "均值列") + sd = snap.number(r, cols["sd"], "标准差列") + group.var = sd * sd if sd is not None and n > 1 else None + else: + total = snap.number(r, cols["sum"], "合计列") + sq = snap.number(r, cols["sumsq"], "平方和列") + if total is not None and n > 0: + group.mean = total / n + if sq is not None and n > 1: + group.var = max(Fraction(0), (sq - total * total / n) / (n - 1)) + else: + v = snap.number(r, cols["value"], "值列") + if v is not None: + group.values.append(v) + if mode == "rows": + for g in groups.values(): + _moments(g) + if by_period: + for w, what in ((params["target"], "本期"), (params["base"], "基期")): + if not any(same_value(w, g.key) for g in groups.values()): + raise CallError("period_not_found", f"期次列「{group_col}」里没有「{w}」：{what}要写 {snap.ref} 里有的期次值") + return groups + + +def _find(groups: Mapping[Any, Group], value: Any, column: str) -> Group: + found = next((g for g in groups.values() if same_value(value, g.key)), None) + if found is None: + shown = "、".join(g.name for g in list(groups.values())[:8]) + raise CallError("unknown_group", f"「{value}」不是「{column}」里的值：要比的组写 {shown} 里的") + return found + + +def _pairs(raw: Mapping[str, Any], params: dict[str, Any], groups: dict[Any, Group], mode: str, dim_label: str, + by_period: bool, group_col: str) -> list[Pair]: + if by_period: + target = _find(groups, params["target"], group_col) + base = _find(groups, params["base"], group_col) + return [Pair(target, base)] + if raw.get("pairs") is not None: + spec = raw.get("pairs") + if not isinstance(spec, list) or not spec or not all(isinstance(p, (list, tuple)) and len(p) == 2 for p in spec): + raise CallError("bad_pairs", "要比的几对写成 [[A, B], …]，A、B 是分组列里的值") + out = [] + for a, b in spec: + ga, gb = _find(groups, a, group_col), _find(groups, b, group_col) + if ga is gb: + raise CallError("bad_pairs", f"「{ga.name}」不能和自己比") + out.append(Pair(ga, gb)) + params["pairs"] = [[p.a.key, p.b.key] for p in out] + return out + if len(groups) < 2: + raise CallError("too_few_groups", f"「{group_col}」里只有 {len(groups)} 组，至少要两组才比得了") + if raw.get("reference") is not None: + ref = _find(groups, raw.get("reference"), group_col) + params["reference"] = ref.key + return [Pair(g, ref) for g in groups.values() if g is not ref] + items = list(groups.values()) + if len(items) == 2: + return [Pair(items[0], items[1])] + rest_name = ph.say("其余各", dim_label) + return [Pair(g, _pool(rest_name, [o for o in items if o is not g], mode)) for g in items] + + +def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair], mode: str, label: str, + criterion: str) -> Any: + alpha = env.threshold("differ_alpha") + min_expected = env.threshold("min_expected") if mode == "rate" else Fraction(0) + for pair in pairs: + _test(pair, mode, min_expected) + hits = _verdict(pairs, alpha) + caveats: list[dict[str, str]] = [] + if mode == "rows": + flipped = _sensitivity(pairs, mode, alpha, min_expected, bool(hits)) + if flipped is not None: + caveats.append(ph.caveat("outlier_sensitive", ph.outlier_text(*flipped))) + hits = _verdict(pairs, alpha) + if mode == "rate" and any(p.test == "fisher" for p in pairs): + caveats.append(ph.caveat("small_sample", ph.small_sample_rate_text(int(env.settings["min_expected"])))) + if mode != "rate": + min_n = int(env.settings["min_n"]) + small = sorted((g for p in pairs for g in (p.a, p.b) if g.n < min_n), key=lambda g: g.n) + if small: + caveats.append(ph.caveat("small_sample", ph.small_sample_mean_text(small[0].name, small[0].n, min_n))) + tested = [p for p in pairs if p.p is not None] + if len(tested) > 1: + caveats.append(ph.caveat("multiple_comparisons", ph.multiple_text(len(tested)))) + fmt = (lambda v: ph.pct(v, signed=False)) if mode == "rate" else (lambda v: ph.amount(v)) + if hits: + code = "differs" + head = hits[0] + main = ph.differ_phrase([ph.differ_pair(p.a.name, p.b.name, label, fmt(p.a_value), fmt(p.b_value), + p.diff > 0) for p in hits], # type: ignore[operator] + len(pairs)) + subjects = [[head.a.name], [head.b.name]] + direction = sign_word(head.diff) # type: ignore[arg-type] + else: + code = "no_evidence" + head = pairs[0] if len(pairs) == 1 else None + sizes = (ph.amount(head.a.n), ph.amount(head.b.n)) if head is not None else None + main = ph.no_evidence_phrase(head.a.name if head else None, head.b.name if head else None, label, sizes, + len(pairs)) + subjects = [[head.a.name], [head.b.name]] if head is not None else [] + direction = None + value_name = "比率" if mode == "rate" else "均值" + rows = [] + for p in pairs: + flags = [] + if p.p is None: + flags.append("样本不够，算不了") + if p.test == "fisher": + flags.append("样本少，用了精确检验") + rows.append([p.a.name, p.b.name, canon(p.a_value), canon(p.b_value), canon(p.diff), + canon(p.ci[0]) if p.ci else None, canon(p.ci[1]) if p.ci else None, str(p.a.n), str(p.b.n), + TESTS.get(p.test or "") or None, canon(p.p), canon(p.p_adj), "、".join(flags) or None]) + table = {"columns": ["a", "b", "a_value", "b_value", "diff", "ci_lo", "ci_hi", "a_n", "b_n", "test", "p", "p_adj", + "flag"], + "labels": {"a": "组", "b": "对照组", "a_value": f"组的{value_name}", "b_value": f"对照组的{value_name}", + "diff": "差", "ci_lo": "差的区间下限", "ci_hi": "差的区间上限", "a_n": "组的样本量", + "b_n": "对照组的样本量", "test": "检验", "p": "p 值", "p_adj": "校正后的 p 值", "flag": "说明"}, + "rows": rows} + shown = head if head is not None else pairs[0] + values = {"a": canon(shown.a_value), "b": canon(shown.b_value), "diff": canon(shown.diff), + "ci_lo": canon(shown.ci[0]) if shown.ci else None, "ci_hi": canon(shown.ci[1]) if shown.ci else None, + "p": canon(shown.p), "p_adj": canon(shown.p_adj), "n_a": str(shown.a.n), "n_b": str(shown.b.n), + "test": shown.test, "pairs": str(len(pairs)), "differs": str(len(hits))} + verdict = {"code": code, "subjects": subjects, "direction": direction} + phrase = ph.with_caveats(main, caveats, skip=("multiple_comparisons",) if len(pairs) > 1 else ()) + return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, phrase=phrase, + caveats=caveats) + + +def _sensitivity(pairs: list[Pair], mode: str, alpha: Fraction, min_expected: Fraction, + differs: bool) -> tuple[bool, bool] | None: + """rows 模式的敏感性：去掉比较的各组里离均值最远的那一个值重算，判定变了交回 (去掉的是不是偏大的那一笔, 现在站不站得住)。""" + everything = [(g, i, v) for g in dict.fromkeys(g for p in pairs for g in (p.a, p.b)) for i, v in enumerate(g.values)] + if len(everything) < 3: + return None + mean = sum((v for _, _, v in everything), Fraction(0)) / len(everything) + group, index, value = max(everything, key=lambda t: abs(t[2] - mean)) + trimmed = [] + for p in pairs: + def cut(g: Group) -> Group: + if g is not group and not (g.key == ("__rest__",) and any(v is value for v in g.values)): + return g + values = list(g.values) + for k, v in enumerate(values): + if v is value: + del values[k] + break + return _moments(Group(g.key, g.name, rows=g.rows, values=values)) + trimmed.append(Pair(cut(p.a), cut(p.b))) + for p in trimmed: + _test(p, mode, min_expected) + now = bool(_verdict(trimmed, alpha)) + _verdict(pairs, alpha) + return (value > mean, now) if now != differs else None + + +__all__ = ["TESTS", "differ"] diff --git a/backend/app/analysis/inputs.py b/backend/app/analysis/inputs.py index 25d82152..88229117 100644 --- a/backend/app/analysis/inputs.py +++ b/backend/app/analysis/inputs.py @@ -198,6 +198,8 @@ class Inputs: out["artifact"] = snap.artifact if snap.dialect: out["dialect"] = snap.dialect + if snap.masked: + out["masked"] = sorted(snap.masked) # 只记列名（复算按它认遮罩列，快照工件里不一定都有） out.update({k: v for k, v in entry.items() if k != "ref"}) if out not in self.records: self.records.append(out) # type: ignore[arg-type] diff --git a/backend/app/analysis/series.py b/backend/app/analysis/series.py new file mode 100644 index 00000000..8bf7267a --- /dev/null +++ b/backend/app/analysis/series.py @@ -0,0 +1,304 @@ +"""放回历史看（细案 01 §3、§4）：波动基线（这个变化大不大、算不算异常）和趋势（是持续在变，还是一次波动）。 + +**参数**：query（一份快照：一条序列；要看某一段就在 SQL 里筛好）、period（期次列）、measure（度量列）；可选 grain（day / +week / month / quarter，缺省按期次值推断，见 analysis/periods）、label。波动基线另有 target（本期，缺省最后一期）、basis +（change 缺省：和比较期比的变化率；level：值本身，给存量、比率用）、lag（和往前第几期比，缺省日数据 7、其余 1；同比写 52 / +12 / 4，历史要相应够长）。 + +**前提**（不满足就算不了，原因照实写）：期次认得出（period_unreadable）、本期完整（同可比性检查的整期：没过完、数据没到、 +SQL 截掉一部分 → partial_target；表的日期范围查不到时只按运行当天判，附一句核对不了）、本期和比较期之间不缺期（target_gap）、 +比较期不为 0（zero_base）、历史够长（history_short）；趋势要最后几期连着（中间缺期 series_gap）。度量格为空的那一期跳过。 + +**波动基线**：历史的中位数 m、中位绝对偏差 MAD，σ̂ = 1.4826·MAD，稳健 z = (x − m)/σ̂；常规波动范围 m ± k_warn·σ̂；全程 +精确的有理数。σ̂ = 0（历史每期都一样）时本期一样判常规、不一样判明显超出。本期的值（不论 basis）是历史各期里最高 / 最低的 +加 record_high / record_low：「历史新高」说的是值本身。为什么不用均值和标准差：促销周这类极端值会把它们拉宽。 + +**趋势**：最后 n 期（期次连着的那一段，trend_min_points ≤ n ≤ max_history + 1）的 Theil–Sen 斜率、Mann–Kendall 双侧 p， +相对斜率 = 斜率 / |序列中位数|。p < trend_alpha 时按斜率的正负判上升、下降。末尾连续同向的期数从最后一期往前数（变化为 0 +断开）。已知局限：Mann–Kendall 假设各期独立，业务序列常有自相关，p 会偏小；所以缺省 n 至少 6、α 取 0.05，判定语只说 +「呈上升趋势」，不说「显著上升」。日数据附一句星期效应。 +""" +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from fractions import Fraction +from typing import Any + +from app.analysis import comparable as cmp +from app.analysis import periods as pd +from app.analysis import phrases as ph +from app.analysis import stats +from app.analysis.common import Env, build, column_label, label_for, need, refuse, sign_word, today_param +from app.analysis.contract import CallError +from app.analysis.inputs import Snap, same_value +from app.analysis.numbers import canon, to_fraction + +#: 稳健标准差 = 1.4826 × MAD（正态时等于标准差） +MAD_SCALE = Fraction(14826, 10000) +#: lag 的上限（周数据同比是 52） +_LAG_MAX = 260 + + +@dataclass +class Series: + snap: Snap + period: str + measure: str + grain: str + label: str + points: list[tuple[pd.Period, Fraction]] # 按期次排好、度量不为空的 + nulls: int + params: dict[str, Any] + + +def _load(raw: Mapping[str, Any], env: Env, used: list[str]) -> Series | Any: + """读出一条序列；算不了时直接交回 refused 的结论。""" + snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号") + period = snap.column(need(raw, "period", "期次列"), "期次列") + measure = snap.column(need(raw, "measure", "度量列"), "度量列") + grain = raw.get("grain") + if grain is not None and grain not in (*pd.GRAINS, "seq"): + raise CallError("bad_grain", "粒度只能写 day、week、month、quarter，或者不写（按期次值推断）") + params: dict[str, Any] = {"query": snap.ref, "period": period, "measure": measure} + label = label_for(env, raw.get("label"), snap, measure) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + criterion = _criterion(env, None, "期") + env.inputs.use(snap, "data", [period, measure]) + if snap.truncated: + return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，序列不完整", + criterion=criterion) + i_period = snap.index(period) + values = [row[i_period] if i_period < len(row) else None for row in snap.rows] + parsed = pd.parse(values, grain) + if parsed is None: + sample = "、".join(dict.fromkeys(ph.key_text([v]) for v in values[:3])) + return refuse(env, params, used, "period_unreadable", + f"期次列「{period}」的值认不出是哪一期（{sample}）：要日期、YYYY-MM、YYYY-Www、YYYY-Qn 或整数序号", + criterion=criterion) + items, grain = parsed + params["grain"] = grain + seen: set[int] = set() + points: list[tuple[pd.Period, Fraction]] = [] + nulls = 0 + for r, item in enumerate(items): + if item.ordinal in seen: + raise CallError("duplicate_rows", ph.say(f"{snap.ref} 里同一期有不止一行：在 SQL 里按 ", + column_label(env, snap, period), " 汇总，每期一行")) + seen.add(item.ordinal) + value = snap.number(r, measure, "度量列") + if value is None: + nulls += 1 + continue + points.append((item, value)) + points.sort(key=lambda p: p[0].ordinal) + return Series(snap, period, measure, grain, label, points, nulls, params) + + +def _criterion(env: Env, n: int | None, unit: str) -> str: + if env.op == "baseline": + return ph.criterion_baseline(env.settings, n, unit) + return ph.criterion_trend(env.settings, n, unit) + + +def _completeness(env: Env, series: Series, item: pd.Period) -> tuple[str, str | None]: + """本期齐不齐（同可比性检查的整期）：(pass / short / unknown, 不齐的那一句)。""" + series.params["today"] = today_param(env) + if item.start is None: + return "unknown", None + span = cmp.period_span("本期", series.snap, series.period, item, series.grain, series.measure, env.facts, env.book) + return cmp.span_coverage(span, env.today, env.book) # type: ignore[arg-type] + + +def _shown_period(item: pd.Period) -> Any: + return item.raw + + +# -------------------------------------------------------------------------- +# 波动基线 +# -------------------------------------------------------------------------- + + +def baseline(raw: Mapping[str, Any], env: Env) -> Any: + used = ["k_warn", "k_alert", "min_history", "max_history"] + loaded = _load(raw, env, used) + if not isinstance(loaded, Series): + return loaded + series, params = loaded, loaded.params + basis = raw.get("basis") or "change" + if basis not in ("change", "level"): + raise CallError("bad_basis", "比法只能写 change（和比较期比的变化率）或 level（值本身）") + lag = raw.get("lag", pd.DEFAULT_LAG[series.grain]) + if isinstance(lag, bool) or not isinstance(lag, int) or not 1 <= lag <= _LAG_MAX: + raise CallError("bad_lag", f"往前比几期要写 1 到 {_LAG_MAX} 的整数（同比：周数据 52、月数据 12、季度 4）") + params.update(basis=basis, lag=lag) + unit, per = pd.COUNT_WORD[series.grain], pd.PER_WORD[series.grain] + criterion = _criterion(env, None, unit) + if not series.points: + return refuse(env, params, used, "history_short", f"只有 0 期历史，至少要 {int(env.settings['min_history'])} 期", + criterion=criterion) + if raw.get("target") is not None: + found = next((p for p in series.points if same_value(raw["target"], p[0].raw)), None) + if found is None: + raise CallError("period_not_found", f"期次列「{series.period}」里没有「{raw['target']}」（或那一期为空）：" + "本期要写序列里有值的一期，不写就是最后一期") + else: + found = series.points[-1] + target, x_level = found + params["target"] = target.raw + caveats: list[dict[str, str]] = [] + status, why = _completeness(env, series, target) + if status == "short": + return refuse(env, params, used, "partial_target", f"{why}，和完整的{per}比没有意义", criterion=criterion) + if status == "unknown" and target.start is not None: + caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE)) + by_ordinal = {p.ordinal: v for p, v in series.points} + before = [(p, v) for p, v in series.points if p.ordinal < target.ordinal] + if basis == "change": + prev = by_ordinal.get(target.ordinal - lag) + if prev is None: + return refuse(env, params, used, "target_gap", + f"本期和它的比较期（往前第 {lag} {'期' if series.grain == 'seq' else unit}）之间缺期，没有比较期的值", + criterion=criterion) + if prev == 0: + return refuse(env, params, used, "zero_base", ph.say("比较期的", series.label, "为 0，算不了变化率"), + criterion=criterion) + x = (x_level - prev) / abs(prev) + history = [(p, (v - by_ordinal[p.ordinal - lag]) / abs(by_ordinal[p.ordinal - lag]), v) for p, v in before + if by_ordinal.get(p.ordinal - lag) not in (None, 0)] + else: + x = x_level + history = [(p, v, v) for p, v in before] + history = history[-int(env.settings["max_history"]):] + n = len(history) + criterion = _criterion(env, n, unit) + if n < int(env.settings["min_history"]): + return refuse(env, params, used, "history_short", f"只有 {n} 期历史，至少要 {int(env.settings['min_history'])} 期", + criterion=criterion) + hv = [h for _, h, _ in history] + m = stats.median(hv) + sigma = MAD_SCALE * stats.mad(hv, m) + k_warn, k_alert = env.threshold("k_warn"), env.threshold("k_alert") + if sigma == 0: + z = None + code = "within_band" if x == m else "far_outside_band" + else: + z = (x - m) / sigma + code = "within_band" if abs(z) < k_warn else "outside_band" if abs(z) < k_alert else "far_outside_band" + lo, hi = m - k_warn * sigma, m + k_warn * sigma + levels = [lv for _, _, lv in history] + if x_level > max(levels): + caveats.append(ph.caveat("record_high", ph.record_text(True, series.label, n, unit))) + elif x_level < min(levels): + caveats.append(ph.caveat("record_low", ph.record_text(False, series.label, n, unit))) + if series.nulls: + caveats.append(ph.caveat("null_values", ph.null_periods_text(series.nulls, series.label))) + current = ph.current_text(x, basis=basis, lag=lag, grain=series.grain) + if sigma == 0: + shown = ph.pct(m) if basis == "change" else ph.amount(m) + main = ph.baseline_flat_phrase(code, series.label, current, n, unit, per, shown) + else: + main = ph.baseline_phrase(code, series.label, current, n, unit, ph.band_text(lo, hi, basis=basis)) + rows = [[p.raw, canon(lv), canon(h) if basis == "change" else None, "历史"] for p, h, lv in history] + rows.append([target.raw, canon(x_level), canon(x) if basis == "change" else None, "本期"]) + table = {"columns": [series.period, "value", "change", "role"], + "labels": {series.period: column_label(env, series.snap, series.period), "value": series.label, + "change": "变化率", "role": "说明"}, + "rows": rows} + values = {"x": canon(x), "value": canon(x_level), "median": canon(m), "band_lo": canon(lo), "band_hi": canon(hi), + "z": canon(z) if z is not None else None, "rank": str(stats.rank_desc(x, hv)), "n_history": str(n), + "hist_min": canon(min(hv)), "hist_max": canon(max(hv))} + verdict = {"code": code, "subjects": [], "direction": sign_word(x - m)} + return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, + phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats) + + +# -------------------------------------------------------------------------- +# 趋势 +# -------------------------------------------------------------------------- + + +def run_length(values: list[Fraction]) -> tuple[int, str | None]: + """末尾连续同向：从最后一期往前数，相邻变化同号的有几个（变化为 0 断开）。""" + changes = [b - a for a, b in zip(values, values[1:])] + if not changes or changes[-1] == 0: + return 0, None + sign = changes[-1] > 0 + count = 0 + for c in reversed(changes): + if c == 0 or (c > 0) != sign: + break + count += 1 + return count, "up" if sign else "down" + + +def trend(raw: Mapping[str, Any], env: Env) -> Any: + used = ["trend_min_points", "trend_alpha", "max_history"] + loaded = _load(raw, env, used) + if not isinstance(loaded, Series): + return loaded + series, params = loaded, loaded.params + unit, per = pd.COUNT_WORD[series.grain], pd.PER_WORD[series.grain] + minimum = int(env.settings["trend_min_points"]) + criterion = _criterion(env, None, unit) + if not series.points: + return refuse(env, params, used, "history_short", f"只有 0 期，至少要 {minimum} 期", criterion=criterion) + last = series.points[-1][0] + caveats: list[dict[str, str]] = [] + status, why = _completeness(env, series, last) + if status == "short": + return refuse(env, params, used, "partial_target", f"{why}，和完整的{per}比没有意义", criterion=criterion) + if status == "unknown" and last.start is not None: + caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE)) + cap = int(env.settings["max_history"]) + 1 + tail = [series.points[-1]] + for point in reversed(series.points[:-1]): + if len(tail) >= cap or point[0].ordinal != tail[-1][0].ordinal - 1: + break + tail.append(point) + tail.reverse() + n = len(tail) + criterion = _criterion(env, n, unit) + if n < minimum: + gap = len(series.points) > n and len(tail) < cap + if gap: + return refuse(env, params, used, "series_gap", + f"最后连着的只有 {n} 期，再往前缺期（至少要连着的 {minimum} 期）", criterion=criterion) + return refuse(env, params, used, "history_short", f"只有 {n} 期，至少要 {minimum} 期", criterion=criterion) + ys = [v for _, v in tail] + xs = [Fraction(i) for i in range(n)] + slope = stats.theil_sen(xs, ys) + med = stats.median(ys) + rel = slope / abs(med) if med != 0 else None + mk = stats.mann_kendall(ys) + alpha = env.threshold("trend_alpha") + p = mk.p if isinstance(mk.p, Fraction) else to_fraction(mk.p) + if p is not None and p < alpha and slope > 0: + code = "rising" + elif p is not None and p < alpha and slope < 0: + code = "falling" + else: + code = "no_trend" + run, direction = run_length(ys) + if series.grain == "day": + caveats.append(ph.caveat("weekday_effect", ph.WEEKDAY_EFFECT)) + if series.nulls: + caveats.append(ph.caveat("null_values", ph.null_periods_text(series.nulls, series.label))) + slope_text = ph.pct(rel) if rel is not None else ph.amount(slope, signed=True) + main = ph.trend_phrase(code, series.label, n, unit, per, slope_text, + (run, direction) if run >= 3 and direction else None) # type: ignore[arg-type] + rows = [[p_.raw, canon(v), canon(v - tail[i - 1][1]) if i else None] for i, (p_, v) in enumerate(tail)] + table = {"columns": [series.period, "value", "change"], + "labels": {series.period: column_label(env, series.snap, series.period), "value": series.label, + "change": "比上一期"}, + "rows": rows} + values = {"slope": canon(slope), "rel_slope": canon(rel) if rel is not None else None, "p": canon(mk.p), + "s": str(mk.s), "n": str(n), "run_length": str(run), "run_direction": direction} + verdict = {"code": code, "subjects": [], "direction": "up" if code == "rising" else "down" if code == "falling" + else None} + return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, + phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats) + + +__all__ = ["MAD_SCALE", "baseline", "run_length", "trend"] diff --git a/backend/tests/analysis_samples.py b/backend/tests/analysis_samples.py new file mode 100644 index 00000000..01fae6b0 --- /dev/null +++ b/backend/tests/analysis_samples.py @@ -0,0 +1,127 @@ +"""分析算子测试用的样例：虚构零售库 shop（地区华东、华南、华北、西南，渠道线上、线下，销售额、订单数、退款），表名、列名、 +数字都是编的。analyze 跑一次 analysis.run，顺手用 recompute 复算一遍、核对逐字段相同（contract.same），所以每个用例都同时 +钉住「复算和运行逐字段相同」。""" +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from datetime import date +from typing import Any + +from app import analysis +from app.analysis.contract import CallError, effective_settings, same +from app.core.artifact_store import canonical_json, content_hash + +#: 测试里的「运行当天」 +TODAY = date(2026, 10, 21) + + +def snap(ref: str, sql: str, columns: Sequence[str], rows: Sequence[Sequence[Any]], *, + types: Mapping[str, str] | None = None, source: str = "shop", truncated: bool = False, + masked: Sequence[str] = (), **extra: Any) -> dict[str, Any]: + """一份输入快照（InputSnapshot 的形状）。artifact 按内容算，复算按它取回。""" + body = {"source": source, "sql": sql, "columns": list(columns), "rows": [list(r) for r in rows], + "truncated": truncated, **({"column_types": dict(types)} if types else {}), **extra} + return {"ref": ref, "artifact": content_hash(canonical_json(body)), "node_id": "q1", "masked": list(masked), **body} + + +def item(value: Any, status: str = "confirmed", source: str = "human") -> dict[str, Any]: + return {"value": value, "source": source, "status": status} + + +def notes(*, columns: Mapping[str, Mapping[str, Any]] | None = None, business_date: str | None = None) -> dict[str, Any]: + """一张表的目录 notes：columns 是 {列: {label / measure: 值或 item(...)}}。""" + out: dict[str, Any] = {} + if columns: + out["columns"] = {c: {k: v if isinstance(v, Mapping) else item(v) for k, v in spec.items()} + for c, spec in columns.items()} + if business_date: + out["business_date"] = item({"column": business_date, "rule": "下单日", "timezone": "Asia/Shanghai"}) + return out + + +#: orders 表的目录：地区、渠道、实收金额（流量）、退款单数 +ORDERS = notes(columns={"region": {"label": "地区"}, "channel": {"label": "渠道"}, + "amount": {"label": "销售额", "measure": "flow"}, + "refund_flag": {"label": "退款单数", "measure": "flow"}, + "order_date": {"label": "下单日"}}, + business_date="order_date") +CATALOGS = {"shop": {"orders": ORDERS}} + + +async def analyze(op: str, mode: str | None = None, *, snaps: Sequence[Mapping[str, Any]], today: date = TODAY, + catalogs: Mapping[str, Any] | None = None, probe: Any = None, facts: Any = None, + parents: Mapping[str, Mapping[str, Any]] | None = None, settings: Mapping[str, Any] | None = None, + hypothesis: str | None = None, check: bool = True, **params: Any) -> dict[str, Any]: + """跑一次 analysis.run；check 时再复算一遍，核对和封存的逐字段相同（hypothesis 除外）。""" + by_ref = {s["ref"]: s for s in snaps} + request: dict[str, Any] = {"op": op, "mode": mode, "params": params} + if hypothesis is not None: + request["hypothesis"] = hypothesis + context: dict[str, Any] = {"today": today, "catalogs": CATALOGS if catalogs is None else catalogs} + if probe is not None: + context["probe"] = probe + if facts is not None: + context["facts"] = facts + result = await analysis.run(request, by_ref.get, settings=effective_settings(settings), context=context, + parents=(parents or {}).get) + if check: + again = recomputed(result, snaps, parents) + assert same(result, again), _diff(result, again) + assert again["hypothesis"] == result["hypothesis"] + return dict(result) + + +def recomputed(result: Mapping[str, Any], snaps: Sequence[Mapping[str, Any]], + parents: Mapping[str, Mapping[str, Any]] | None = None) -> dict[str, Any]: + """按结论里记下的复算：快照按 artifact 取回（只交快照内容本身，不交入口加的出处），上一层结论按内容哈希取回。""" + by_artifact = {s["artifact"]: {k: v for k, v in s.items() if k not in ("ref", "artifact", "node_id", "masked")} + for s in snaps} + by_hash = {content_hash(canonical_json(r)): r for r in (parents or {}).values()} + return dict(analysis.recompute(result, by_artifact.get, load_result=by_hash.get)) # type: ignore[arg-type] + + +def _diff(a: Mapping[str, Any], b: Mapping[str, Any]) -> str: + keys = sorted(k for k in set(a) | set(b) if a.get(k) != b.get(k)) + return "；".join(f"{k}: {json.dumps(a.get(k), ensure_ascii=False)} ≠ {json.dumps(b.get(k), ensure_ascii=False)}" + for k in keys) + + +async def call_error(op: str, mode: str | None = None, **kwargs: Any) -> CallError: + """跑一次，期望是调用错误。""" + try: + await analyze(op, mode, check=False, **kwargs) + except CallError as e: + return e + raise AssertionError("应当是调用错误") + + +def fixed_prober(ranges: Mapping[tuple[str, str], tuple[str | None, str | None]] | None = None, *, + bounds: Mapping[str, str] | None = None, at: str = "2026-10-21T09:00:00") -> Any: + """测试用的探查器：ranges 是 {(表, 列): (最早, 最晚)}，bounds 是 {界的 SQL: 日期}；没写的查不到。""" + + class Prober: + def __init__(self) -> None: + self.calls: list[tuple[str, ...]] = [] + + async def __call__(self, source: str, table: str, column: str) -> dict[str, Any] | None: + self.calls.append((source, table, column)) + found = (ranges or {}).get((table, column)) + if found is None: + return None + return {"source": source, "table": table, "column": column, "min": found[0], "max": found[1], + "basis": "live", "at": at} + + async def bounds(self, source: str, expr: str) -> dict[str, Any] | None: + self.calls.append((source, expr)) + value = (bounds or {}).get(expr) + return None if value is None else {"source": source, "expr": expr, "value": value, "basis": "live", "at": at} + + def dialect(self, source: str) -> str | None: + return "sqlite" + + return Prober() + + +__all__ = ["CATALOGS", "ORDERS", "TODAY", "analyze", "call_error", "fixed_prober", "item", "notes", "recomputed", + "snap"] diff --git a/backend/tests/test_analysis_comparable.py b/backend/tests/test_analysis_comparable.py new file mode 100644 index 00000000..bf9bb3d8 --- /dev/null +++ b/backend/tests/test_analysis_comparable.py @@ -0,0 +1,182 @@ +"""可比性检查（app/analysis/comparable.py，细案 01 §6）：只差日期 / 差了别的筛选 / 结构不同、窗口不等（流量不可比、比率只提醒）、 +本期不完整、周末天数不同、探查失败退回剖析事实、都没有就核对不了。每个用例都顺带核对复算和运行逐字段相同（复算用记下的 +probes，不现查）。""" +from __future__ import annotations + +from analysis_samples import analyze, fixed_prober, snap +from app.analysis import comparable_reasons + +WINDOW = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= '{a}' AND order_date < '{b}'" +RANGE = {("orders", "order_date"): ("2025-01-01", "2026-10-20")} + + +def period(ref, a, b, sql=WINDOW, **kw): + return snap(ref, sql.format(a=a, b=b), ["gmv"], [[100]], types={"gmv": "number"}, **kw) + + +async def compare(base, target, *, probe=None, **params): + params = {"base_query": base["ref"], "target_query": target["ref"], **params} + return await analyze("comparable", snaps=[base, target], probe=probe, **params) + + +def checks(result): + return {r[0]: r[1] for r in result["table"]["rows"]} + + +async def test_two_weeks_that_differ_only_in_dates_are_comparable(): + result = await compare(period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19"), + probe=fixed_prober(RANGE), measure="gmv") + assert result["method"] == "comparable@1" and result["mode"] is None + assert result["verdict"] == {"code": "comparable", "subjects": [], "direction": None} + assert result["phrase"] == "两期可比" + assert checks(result) == {"同口径": "通过", "窗口长度": "通过", "整期": "通过", "星期构成": "通过", "数据问题": "通过"} + assert result["values"]["base_days"] == "7" and result["values"]["target_start"] == "2026-10-12" + assert result["values"]["window"] == "pass" and result["settings"] == {} + assert result["params"]["today"] == "2026-10-21" and result["params"]["measure"] == "gmv" + assert [p["column"] for p in result["probes"]] == ["order_date"] + assert result["table"]["columns"] == ["check", "status", "base", "target", "note"] + + +async def test_other_literals_differing_mean_two_segments_not_two_periods(): + sql = WINDOW + " AND region = '{r}'" + base = snap("Q1", sql.format(a="2026-10-05", b="2026-10-12", r="华南"), ["gmv"], [[100]]) + target = snap("Q2", sql.format(a="2026-10-12", b="2026-10-19", r="华东"), ["gmv"], [[100]]) + result = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv") + assert result["verdict"]["code"] == "incomparable" + assert result["phrase"] == "两期不可比：两边除了日期，筛选的还不一样" + assert comparable_reasons(result) == ["两边除了日期，筛选的还不一样"] + + +async def test_a_different_structure_says_where_it_differs(): + base = period("Q1", "2026-10-05", "2026-10-12") + target = period("Q2", "2026-10-12", "2026-10-19", sql=WINDOW + " AND status = 'paid'") + result = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv") + assert result["verdict"]["code"] == "incomparable" + assert result["phrase"] == "两期不可比：两份查询的口径结构不同（差在筛选条件）" + other = await compare(base, period("Q2", "2026-10-12", "2026-10-19", source="crm"), measure="gmv") + assert "两份查询来自不同的数据源" in other["phrase"] + + +async def test_unequal_windows_fail_for_flows_and_only_warn_otherwise(): + base, target = period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-15") + flow = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv") # SUM(amount)：目录标的流量 + assert flow["verdict"]["code"] == "incomparable" and checks(flow)["窗口长度"] == "不通过" + assert "两段的天数不同：7 天对 3 天" in flow["phrase"] + avg_sql = "SELECT AVG(amount) AS aov FROM orders WHERE order_date >= '{a}' AND order_date < '{b}'" + ratio = await compare(period("Q1", "2026-10-05", "2026-10-12", sql=avg_sql), + period("Q2", "2026-10-12", "2026-10-15", sql=avg_sql), probe=fixed_prober(RANGE), + measure="gmv") + assert ratio["verdict"]["code"] == "conditional" and checks(ratio)["窗口长度"] == "提醒" + unlabeled = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv", catalogs={}) + assert unlabeled["verdict"]["code"] == "conditional" + assert any(c["code"] == "measure_unlabeled" for c in unlabeled["caveats"]) + + +async def test_whole_months_count_as_the_same_length(): + result = await compare(period("Q1", "2026-08-01", "2026-09-01"), period("Q2", "2026-09-01", "2026-10-01"), + probe=fixed_prober(RANGE), measure="gmv") + assert result["verdict"]["code"] == "comparable" and result["values"]["base_days"] == "31" + + +async def test_an_unfinished_period_is_not_a_whole_period(): + result = await compare(period("Q1", "2026-10-12", "2026-10-19"), period("Q2", "2026-10-19", "2026-10-26"), + probe=fixed_prober(RANGE), measure="gmv") + assert result["verdict"]["code"] == "incomparable" + assert result["phrase"] == "两期不可比：本期数据只到 2026-10-20，还差 5 天" + late = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-16")}) + behind = await compare(period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19"), + probe=late, measure="gmv") + assert behind["phrase"] == "两期不可比：本期数据只到 2026-10-16，还差 2 天" + + +async def test_weekend_composition_of_short_windows(): + # 周一到周三 对 周五到周日 + result = await compare(period("Q1", "2026-10-05", "2026-10-08"), period("Q2", "2026-10-16", "2026-10-19"), + probe=fixed_prober(RANGE), measure="gmv") + assert result["verdict"]["code"] == "conditional" + assert result["phrase"] == "两期有条件可比：两段的周末天数不同：0 天对 2 天" + + +async def test_a_failed_probe_falls_back_to_profile_facts_then_says_it_cannot_check(): + facts = {"shop": {"orders": {"at": "2026-10-20T03:00:00", "date_ranges": [ + {"column": "order_date", "min": "2025-01-01", "max": "2026-10-19"}]}}} + base, target = period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19") + fallback = await compare(base, target, probe=fixed_prober({}), facts=facts, measure="gmv") + assert fallback["verdict"]["code"] == "comparable" and fallback["phrase"] == "两期可比" + assert fallback["probes"] == [{"source": "shop", "table": "orders", "column": "order_date", "min": "2025-01-01", + "max": "2026-10-19", "basis": "profile", "at": "2026-10-20"}] + nothing = await compare(base, target, measure="gmv") + assert nothing["verdict"]["code"] == "comparable" + assert nothing["phrase"] == "两期可比，整期核对不了" and nothing["probes"] == [] + + +async def test_bounds_computed_from_the_run_day_are_evaluated_in_the_database_and_recorded(): + sql = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', '{a}') AND order_date < date('now', '{b}')" + base = snap("Q1", sql.format(a="-16 days", b="-9 days"), ["gmv"], [[100]]) + target = snap("Q2", sql.format(a="-9 days", b="-2 days"), ["gmv"], [[90]]) + bounds = {"DATE('now', '-16 days')": "2026-10-05", "DATE('now', '-9 days')": "2026-10-12", + "DATE('now', '-2 days')": "2026-10-19"} + probe = fixed_prober(RANGE, bounds=bounds) + result = await compare(base, target, probe=probe, measure="gmv") + assert result["verdict"]["code"] == "comparable" + assert checks(result)["同口径"] == "通过" # 只差按运行当天现算的界 + assert result["values"]["base_start"] == "2026-10-05" and result["values"]["target_end"] == "2026-10-18" + exprs = [p["expr"] for p in result["probes"] if "expr" in p] + assert exprs == ["DATE('now', '-16 days')", "DATE('now', '-9 days')", "DATE('now', '-2 days')"] + assert all(p["basis"] == "live" for p in result["probes"]) + # 界算不出来：窗口核对不了，不判不可比 + blind = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv") + assert blind["verdict"]["code"] == "comparable" and "窗口长度" in blind["phrase"] + + +async def test_data_problems_on_the_snapshots(): + base = period("Q1", "2026-10-05", "2026-10-12") + empty = period("Q2", "2026-10-12", "2026-10-19", no_data=[{"columns": ["gmv"], "table": "orders", + "column": "order_date", "message": "没有数据"}]) + result = await compare(base, empty, probe=fixed_prober(RANGE), measure="gmv") + assert result["verdict"]["code"] == "incomparable" and "本期查询的区间里这张表没有数据" in result["phrase"] + checked = period("Q2", "2026-10-12", "2026-10-19", checks=[{"code": "fanout_sum", "level": "error", + "message": "关联订单明细后对订单金额求和，会重复计算"}]) + warned = await compare(base, checked, probe=fixed_prober(RANGE), measure="gmv") + assert warned["verdict"]["code"] == "conditional" + assert "本期查询的 SQL 检查有错误：关联订单明细后对订单金额求和，会重复计算" in warned["phrase"] + + +async def test_one_snapshot_grouped_by_period_is_naturally_the_same_query(): + sql = "SELECT order_date AS day, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-01' GROUP BY 1" + rows = [[f"2026-10-{d:02d}", 100] for d in range(1, 20)] + s = snap("Q1", sql, ["day", "gmv"], rows, types={"day": "text", "gmv": "number"}) + probe = fixed_prober(RANGE) + same_weekday = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="day", base="2026-10-05", + target="2026-10-12") + assert same_weekday["verdict"]["code"] == "comparable" and same_weekday["params"]["measure"] == "gmv" + assert same_weekday["params"]["grain"] == "day" + weekend = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="day", base="2026-10-09", + target="2026-10-10") + assert weekend["phrase"] == "两期有条件可比：两段的周末天数不同：0 天对 1 天" + # 期次有一截落在 SQL 的时间条件之外：基期被截掉了；本期（10 月）还没过完 + monthly_rows = [["2026-09-01", 100], ["2026-10-01", 90]] + month = snap("Q1", "SELECT DATE(order_date, 'start of month') AS month, SUM(amount) AS gmv FROM orders " + "WHERE order_date >= '2026-09-05' GROUP BY 1", ["month", "gmv"], monthly_rows) + partial = await analyze("comparable", snaps=[month], probe=probe, query="Q1", period="month", base="2026-09-01", + target="2026-10-01", measure="gmv") + assert partial["verdict"]["code"] == "incomparable" + assert "基期数据从 2026-09-05 才开始，缺前 4 天" in partial["phrase"] + assert "本期数据只到 2026-10-20，还差 11 天" in partial["phrase"] + + +async def test_reasons_are_split_one_sentence_each(): + result = await compare(period("Q1", "2026-10-05", "2026-10-08"), period("Q2", "2026-10-16", "2026-10-21"), + probe=fixed_prober(RANGE), measure="gmv") + reasons = comparable_reasons(result) + assert "两段的天数不同：3 天对 5 天" in reasons and "本期数据只到 2026-10-20，还差 0 天" not in reasons + assert all("；" not in r for r in reasons) + assert comparable_reasons({"table": None}) == [] + + +async def test_a_masked_date_column_is_neither_used_nor_probed(): + probe = fixed_prober(RANGE) + result = await compare(period("Q1", "2026-10-05", "2026-10-12", masked=["order_date"]), + period("Q2", "2026-10-12", "2026-10-19", masked=["order_date"]), probe=probe, measure="gmv") + assert probe.calls == [] and result["probes"] == [] + assert result["verdict"]["code"] == "comparable" and "窗口长度" in result["phrase"] diff --git a/backend/tests/test_analysis_decompose.py b/backend/tests/test_analysis_decompose.py new file mode 100644 index 00000000..8e34a692 --- /dev/null +++ b/backend/tests/test_analysis_decompose.py @@ -0,0 +1,420 @@ +"""变化拆解（app/analysis/decompose.py，细案 01 §2）：三种恒等式逐位成立、辛普森、抵消、不闭合、下钻闭合与超深、新出现 / +消失的段、比率当可加的调用错误……每个用例都顺带核对复算和运行逐字段相同（tests/analysis_samples.analyze）。""" +from __future__ import annotations + +import math +import random +from decimal import Decimal +from fractions import Fraction + +import pytest + +from analysis_samples import analyze, call_error, fixed_prober, notes, snap +from app.analysis.decompose import lmdi, ratio_effects + +SQL = "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week" +TYPES = {"region": "text", "week": "text", "gmv": "number"} +WEEKLY = [["华东", "2026-W41", 820], ["华东", "2026-W42", 700], ["华北", "2026-W41", 600], ["华北", "2026-W42", 560], + ["华南", "2026-W41", 700], ["华南", "2026-W42", 666], ["西南", "2026-W41", 100], ["西南", "2026-W42", 100]] + + +def weekly(rows=WEEKLY, **kw): + return snap("Q2", SQL, ["region", "week", "gmv"], rows, types=TYPES, **kw) + + +async def additive(rows=WEEKLY, *, snaps=None, **params): + params = {"query": "Q2", "by": ["region"], "period": "week", "base": "2026-W41", "target": "2026-W42", + "measure": "gmv", **params} + return await analyze("decompose", "additive", snaps=snaps or [weekly(rows)], **params) + + +def _rows(base: dict, target: dict) -> list[list]: + return [[k, "2026-W41", v] for k, v in base.items()] + [[k, "2026-W42", v] for k, v in target.items()] + + +# -------------------------------------------------------------------------- +# additive +# -------------------------------------------------------------------------- + + +async def test_a_driver_is_named_with_its_share_and_the_whole_shape_is_filled(): + result = await additive(label="销售额", hypothesis="华东拖累") + assert result["method"] == "decompose.additive@1" and result["status"] == "ok" + assert result["verdict"] == {"code": "driver", "subjects": [["华东"]], "direction": "down"} + assert result["phrase"] == "销售额下降 194（-8.7%），主要来自华东（-120，占 61.9%）" + assert result["values"]["delta_total"] == "-194" and result["values"]["top_key"] == "华东" + assert result["values"]["top_share"] == "0.618556701031" and result["values"]["segments"] == "4" + table = result["table"] + assert table["columns"] == ["region", "base", "target", "delta", "pct", "share", "flag"] + assert table["labels"]["region"] == "地区" and table["labels"]["share"] == "占变化的比重" + assert [r[0] for r in table["rows"]] == ["华东", "华北", "华南", "西南"] # 按变化的绝对值从大到小 + assert result["settings"] == {"main_share": 0.5, "concentration": 0.8, "offset_ratio": 0.25, "flat_pct": 0.005, + "table_top_k": 10} + assert result["inputs"] == [{"ref": "Q2", "artifact": weekly()["artifact"], "role": "data", + "columns": ["region", "week", "gmv"], "rows": "all"}] + assert result["hypothesis"] == "华东拖累" and result["probes"] == [] + assert result["criterion"].endswith("以各段之和为总数") + assert {c["code"] for c in result["caveats"]} == {"headline_basis"} + assert result["params"]["catalog"] == {"labels": {"region": "地区"}, "measures": {"gmv": "flow"}} + + +async def test_the_segments_add_up_to_the_total_digit_for_digit(): + rng = random.Random(7) + regions = ["华东", "华南", "华北", "西南", "东北", "西北"] + for _ in range(20): + base = {r: round(rng.uniform(0, 5000), rng.choice([0, 2, 3])) for r in regions} + target = {r: round(rng.uniform(0, 5000), rng.choice([0, 2, 3])) for r in regions} + result = await additive(_rows(base, target)) + rows = result["table"]["rows"] + assert sum(Decimal(r[3]) for r in rows) == Decimal(result["values"]["delta_total"]) + assert sum(Decimal(r[1]) for r in rows) == Decimal(result["values"]["base_total"]) + assert sum(Decimal(r[2]) for r in rows) == Decimal(result["values"]["target_total"]) + assert Decimal(result["values"]["target_total"]) - Decimal(result["values"]["base_total"]) == \ + Decimal(result["values"]["delta_total"]) + assert result["values"]["gross_up"] and result["values"]["gross_down"] + + +async def test_concentrated_and_dispersed(): + base = {f"店{i}": 1000 for i in range(10)} + target = {**base, "店0": 1450, "店1": 1400, "店2": 1050, "店3": 1050, "店4": 1050} + result = await additive(_rows(base, target), by="region") + assert result["verdict"]["code"] == "concentrated" + assert result["verdict"]["subjects"] == [["店0"], ["店1"]] + assert result["values"]["top_k"] == "2" and result["values"]["top_k_share"] == "0.85" + assert "集中在店0、店1（合计占 85.0%）" in result["phrase"] + spread = await additive(_rows({r: 1000 for r in "甲乙丙丁戊"}, {r: 1100 for r in "甲乙丙丁戊"})) + assert spread["verdict"] == {"code": "dispersed", "subjects": [], "direction": "up"} + assert "分散在各地区，没有哪一段占到一半" in spread["phrase"] + + +async def test_offsetting_segments_are_said_and_a_share_can_pass_one_hundred_percent(): + result = await additive(_rows({"华东": 1000, "华北": 500, "华南": 500}, {"华东": 850, "华北": 540, "华南": 490})) + assert result["values"]["delta_total"] == "-120" + assert result["verdict"]["code"] == "driver" and result["values"]["top_share"] == "1.25" + offset = next(c for c in result["caveats"] if c["code"] == "offset") + assert offset["text"] == "下降的段合计 -160，上升的段合计 +40，互相抵消了一部分" + assert result["phrase"] == "销售额下降 120（-6.0%），主要来自华东（-150，占 125.0%）；下降的段合计 -160，上升的段合计 +40，互相抵消了一部分" + + +async def test_flat_with_offsets_says_what_moved_inside(): + result = await additive(_rows({"华东": 1000, "华北": 1000, "华南": 220}, {"华东": 1120, "华北": 882, "华南": 220})) + assert result["verdict"] == {"code": "flat", "subjects": [], "direction": None} + assert result["phrase"] == "销售额基本持平（+0.1%），内部此消彼长：华东 +120、华北 -118" + + +async def test_a_headline_that_does_not_close_stops_the_concentration_verdict(): + head = snap("Q1", "SELECT week, SUM(amount) AS gmv FROM orders GROUP BY week", ["week", "gmv"], + [["2026-W42", 2050], ["2026-W41", 2220]], types={"week": "text", "gmv": "number"}) + result = await additive(snaps=[weekly(), head], headline={"base": "Q1.r1.gmv", "target": "Q1.r0.gmv"}) + assert result["verdict"] == {"code": "unclosed", "subjects": [], "direction": None} + assert result["values"]["gap"] == "24" and result["values"]["top_share"] is None + assert all(r[5] is None for r in result["table"]["rows"]) # 不给占比 + assert result["phrase"] == "按地区拆开后各段之和和总数对不上（差 24），不判断变化来自哪里" + assert [i["role"] for i in result["inputs"]] == ["headline_base", "headline_target", "data"] + assert result["inputs"][0] == {"ref": "Q1.r1.gmv", "artifact": head["artifact"], "role": "headline_base", + "row": 1, "column": "gmv", "value": "2220"} + assert "各段之和要和总数对得上" in result["criterion"] + + +async def test_a_rounded_headline_closes_within_its_written_precision(): + rows = [["华东", "2026-W41", "820.25"], ["华东", "2026-W42", "700.12"], ["华北", "2026-W41", "1399.9"], + ["华北", "2026-W42", "1325.84"]] + for written, closes in (("2220.2", True), ("2220", True), ("2221", False)): + head = snap("Q1", "SELECT SUM(amount) AS gmv FROM orders", ["gmv_base", "gmv"], [[written, "2025.96"]]) + result = await additive(rows, snaps=[weekly(rows), head], headline=["Q1.r0.gmv_base", "Q1.r0.gmv"]) + assert (result["verdict"]["code"] != "unclosed") is closes, written + + +async def test_drilling_down_checks_closure_against_the_parent_segment(): + top = await additive() + province = snap("Q4", "SELECT province, week, SUM(amount) AS gmv FROM orders WHERE region = '华东' GROUP BY 1, 2", + ["province", "week", "gmv"], + [["沪", "2026-W41", 500], ["沪", "2026-W42", 410], ["苏", "2026-W41", 320], ["苏", "2026-W42", 290]]) + params = {"query": "Q4", "by": ["province"], "period": "week", "base": "2026-W41", "target": "2026-W42", + "measure": "gmv", "parent": {"of": "A3", "key": ["华东"]}} + child = await analyze("decompose", "additive", snaps=[province], parents={"A3": top}, **params) + assert child["verdict"]["code"] == "driver" and child["verdict"]["subjects"] == [["沪"]] + assert child["params"]["depth"] == 1 and child["params"]["parent"] == {"of": "A3", "key": ["华东"]} + assert child["inputs"][0]["role"] == "parent" and child["inputs"][0]["ref"] == "A3" + assert "max_drill_depth" in child["settings"] + assert {c["code"] for c in child["caveats"]} == set() # 有 parent 就不是「以各段之和为总数」 + short = snap("Q4", province["sql"], ["province", "week", "gmv"], + [["沪", "2026-W41", 500], ["沪", "2026-W42", 410], ["苏", "2026-W41", 300], ["苏", "2026-W42", 290]]) + unclosed = await analyze("decompose", "additive", snaps=[short], parents={"A3": top}, **params) + assert unclosed["verdict"]["code"] == "unclosed" and unclosed["values"]["gap"] == "20" + # 超深：上一层已经是第 1 层，最多 1 层 + error = await call_error("decompose", "additive", snaps=[province], parents={"A3": child}, + settings={"max_drill_depth": 1}, **{**params, "parent": {"of": "A3", "key": ["沪"]}}) + assert error.code == "too_deep" and "最多 1 层" in error.message + missing = await call_error("decompose", "additive", snaps=[province], parents={"A3": top}, + **{**params, "parent": {"of": "A3", "key": ["东北"]}}) + assert missing.code == "parent_key_missing" + absent = await call_error("decompose", "additive", snaps=[province], **params) + assert absent.code == "parent_missing" + + +async def test_new_and_vanished_segments_and_empty_cells_are_flagged(): + rows = [["华东", "2026-W41", 800], ["华东", "2026-W42", 700], ["西南", "2026-W42", 90], + ["东北", "2026-W41", 40], ["华北", "2026-W41", None], ["华北", "2026-W42", 30]] + result = await additive(rows) + flags = {r[0]: r[6] for r in result["table"]["rows"]} + assert flags == {"华东": None, "西南": "新出现", "东北": "消失", "华北": "为空"} + null = next(c for c in result["caveats"] if c["code"] == "null_values") + assert null["text"] == "华北的销售额为空，按 0 算" + assert next(r for r in result["table"]["rows"] if r[0] == "西南")[4] is None # 基期为 0 不给变化率 + + +async def test_two_dimension_keys(): + sql = "SELECT region, channel, week, SUM(amount) AS gmv FROM orders GROUP BY 1, 2, 3" + rows = [["华东", "线上", "W1", 500], ["华东", "线上", "W2", 300], ["华东", "线下", "W1", 300], ["华东", "线下", "W2", 310], + ["华北", "线上", "W1", 200], ["华北", "线上", "W2", 205]] + s = snap("Q2", sql, ["region", "channel", "week", "gmv"], rows) + result = await analyze("decompose", "additive", snaps=[s], query="Q2", by=["region", "channel"], period="week", + base="W1", target="W2", measure="gmv") + assert result["verdict"]["subjects"] == [["华东", "线上"]] + assert "主要来自华东·线上" in result["phrase"] + + +async def test_ratio_and_non_additive_measures_are_call_errors(): + avg = snap("Q2", "SELECT region, week, AVG(amount) AS aov FROM orders GROUP BY 1, 2", ["region", "week", "aov"], WEEKLY) + error = await call_error("decompose", "additive", snaps=[avg], query="Q2", by="region", period="week", + base="2026-W41", target="2026-W42", measure="aov") + assert error.code == "ratio_as_additive" and "改用比率拆解" in error.message + rate_sql = "SELECT region, week, SUM(refund_rate) AS r FROM orders GROUP BY 1, 2" + labeled = {"shop": {"orders": notes(columns={"refund_rate": {"measure": "ratio"}})}} + error = await call_error("decompose", "additive", snaps=[snap("Q2", rate_sql, ["region", "week", "r"], WEEKLY)], + catalogs=labeled, query="Q2", by="region", period="week", base="2026-W41", + target="2026-W42", measure="r") + assert error.code == "ratio_as_additive" + # 只是推断（没人确认）的照算，写附带说明 + guessed = {"shop": {"orders": notes(columns={"refund_rate": {"measure": {"value": "ratio", "source": "llm", + "status": "proposed"}}})}} + result = await analyze("decompose", "additive", snaps=[snap("Q2", rate_sql, ["region", "week", "r"], WEEKLY)], + catalogs=guessed, query="Q2", by="region", period="week", base="2026-W41", + target="2026-W42", measure="r") + note = next(c for c in result["caveats"] if c["code"] == "measure_unlabeled") + assert note["text"] == "目录里「r」是比率只是推断，没人确认，按可以按段相加处理" + unlabeled = await additive(snaps=[weekly()], catalogs={}) + assert any(c["code"] == "measure_unlabeled" and "目录没标「gmv」的度量类型" in c["text"] for c in unlabeled["caveats"]) + + +async def test_call_errors_tell_the_agent_how_to_fix_the_call(): + dup = WEEKLY + [["华东", "2026-W42", 5]] + error = await call_error("decompose", "additive", snaps=[weekly(dup)], query="Q2", by="region", period="week", + base="2026-W41", target="2026-W42", measure="gmv") + assert error.code == "duplicate_rows" and "在 SQL 里按 地区、week 汇总" in error.message + masked = await call_error("decompose", "additive", snaps=[weekly(masked=["Region"])], query="Q2", by="region", + period="week", base="2026-W41", target="2026-W42", measure="gmv") + assert masked.code == "masked_column" + missing = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="region", period="week", + base="2026-W40", target="2026-W42", measure="gmv") + assert missing.code == "period_not_found" and "2026-W41" in missing.message + column = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="area", period="week", + base="2026-W41", target="2026-W42", measure="gmv") + assert column.code == "unknown_column" and "region、week、gmv" in column.message + text = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="gmv", period="week", + base="2026-W41", target="2026-W42", measure="region") + assert text.code == "not_numeric" + unknown = await call_error("decompose", "additive", snaps=[weekly()], query="Q9", by="region", period="week", + base="2026-W41", target="2026-W42", measure="gmv") + assert unknown.code == "unknown_query" + same_column = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="week", period="week", + base="2026-W41", target="2026-W42", measure="gmv") + assert same_column.code == "bad_by" + + +async def test_truncated_snapshots_and_single_segments_are_refused(): + result = await additive(snaps=[weekly(truncated=True)]) + assert result["status"] == "refused" and result["reason_code"] == "truncated" + assert result["reason"] == "Q2 只取回了前 8 行，各段之和不是完整的和" + assert result["phrase"] == "变化拆解没有做成：Q2 只取回了前 8 行，各段之和不是完整的和" + assert result["verdict"] is None and result["table"] is None and result["values"] == {} + one = await additive([["华东", "2026-W41", 1], ["华东", "2026-W42", 2]]) + assert one["reason_code"] == "too_few_segments" + + +async def test_two_snapshots_check_comparability_first(): + base = snap("Q1", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' AND " + "order_date < '2026-10-12' GROUP BY region", ["region", "gmv"], + [["华东", 820], ["华北", 600], ["华南", 700]]) + target = snap("Q2", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-12' AND " + "order_date < '2026-10-19' GROUP BY region", ["region", "gmv"], + [["华东", 700], ["华北", 560], ["华南", 666]]) + probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")}) + result = await analyze("decompose", "additive", snaps=[base, target], probe=probe, base_query="Q1", + target_query="Q2", by="region", measure="gmv") + assert result["verdict"]["code"] == "driver" and result["params"]["today"] == "2026-10-21" + assert not any(c["code"] == "comparability" for c in result["caveats"]) + assert [i["role"] for i in result["inputs"]] == ["base", "target"] + assert result["probes"] == [{"source": "shop", "table": "orders", "column": "order_date", "min": "2025-01-01", + "max": "2026-10-20", "basis": "live", "at": "2026-10-21T09:00:00"}] + # 本期没过完：照样拆，判定后附上可比性结论 + partial = snap("Q2", target["sql"].replace("'2026-10-19'", "'2026-10-25'"), ["region", "gmv"], target["rows"]) + result = await analyze("decompose", "additive", snaps=[base, partial], probe=probe, base_query="Q1", + target_query="Q2", by="region", measure="gmv") + note = next(c for c in result["caveats"] if c["code"] == "comparability") + assert note["text"].startswith("两期不可比：") and "本期数据只到 2026-10-20，还差 4 天" in note["text"] + assert result["verdict"]["code"] == "driver" and note["text"] in result["phrase"] + # 口径结构不同：不拆 + other = snap("Q2", "SELECT region, SUM(amount) AS gmv FROM orders JOIN refunds USING (order_id) WHERE order_date " + ">= '2026-10-12' GROUP BY region", ["region", "gmv"], target["rows"]) + refused = await analyze("decompose", "additive", snaps=[base, other], probe=probe, base_query="Q1", + target_query="Q2", by="region", measure="gmv") + assert refused["reason_code"] == "incomparable_sides" and refused["reason"].startswith("两份查询的口径结构不同") + + +# -------------------------------------------------------------------------- +# ratio +# -------------------------------------------------------------------------- + +RATIO_SQL = "SELECT channel, week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY 1, 2" + + +def _ratio_snap(rows): + return snap("Q1", RATIO_SQL, ["channel", "week", "refunds", "orders"], rows) + + +async def ratio(rows, **params): + params = {"query": "Q1", "by": "channel", "period": "week", "base": "W41", "target": "W42", + "numerator": "refunds", "denominator": "orders", **params} + return await analyze("decompose", "ratio", snaps=[_ratio_snap(rows)], **params) + + +def test_mix_plus_rate_effects_equal_the_ratio_change_exactly(): + rng = random.Random(11) + for _ in range(200): + segs = [] + for _ in range(rng.randint(2, 7)): + d_b, d_t = rng.choice([0, rng.randint(1, 5000)]), rng.randint(1, 5000) + segs.append((Fraction(rng.randint(0, d_b)) if d_b else Fraction(0), Fraction(d_b), + Fraction(rng.randint(0, d_t)), Fraction(d_t))) + if sum(s[1] for s in segs) == 0: + continue + R, R2, effects = ratio_effects(segs) + assert sum((e[4] for e in effects), Fraction(0)) + sum((e[5] for e in effects), Fraction(0)) == R2 - R + assert sum((e[0] for e in effects), Fraction(0)) == 1 == sum((e[1] for e in effects), Fraction(0)) + + +async def test_simpson_each_channel_improves_but_the_mix_makes_the_whole_worse(): + rows = [["线上", "W41", 50, 500], ["线上", "W42", 90, 1000], ["线下", "W41", 10, 1000], ["线下", "W42", 4, 500]] + result = await ratio(rows, label="退款率") + assert result["verdict"] == {"code": "mix_driven", "subjects": [["线上"]], "direction": "up"} + assert result["phrase"].startswith("退款率上升 2.3 个百分点（4.0% 到 6.3%），主要来自结构变化：线上占比从 33.3% 到 66.7%") + simpson = next(c for c in result["caveats"] if c["code"] == "simpson") + assert simpson["text"] == "各段的退款率大多在下降，整体却上升：变化来自结构，线上占比从 33.3% 升到 66.7%" + values = result["values"] + assert Fraction(values["mix_total"]) + Fraction(values["rate_total"]) == pytest.approx(Fraction(values["delta"])) + assert result["table"]["columns"][:5] == ["channel", "n_base", "n_target", "d_base", "d_target"] + + +async def test_rate_driven_both_and_small_denominators(): + rows = [["线上", "W41", 50, 1000], ["线上", "W42", 80, 1000], ["线下", "W41", 20, 1000], ["线下", "W42", 26, 1000]] + result = await ratio(rows, label="退款率") + assert result["verdict"]["code"] == "rate_driven" and result["verdict"]["subjects"] == [["线上"]] + assert result["phrase"] == "退款率上升 1.8 个百分点（3.5% 到 5.3%），主要来自各段自身的变化，线上影响最大" + tiny = [["线上", "W41", 2, 20], ["线上", "W42", 3, 25], ["线下", "W41", 20, 1000], ["线下", "W42", 30, 1000]] + small = await ratio(tiny, label="退款率") + note = next(c for c in small["caveats"] if c["code"] == "small_denominator") + assert note["text"] == "线上的 orders 不到 30，比率波动大，不参与多数判断" + assert "分母小" in next(r for r in small["table"]["rows"] if r[0] == "线上")[-1] + + +async def test_ratio_refusals(): + zero = [["线上", "W41", 0, 0], ["线上", "W42", 3, 100], ["线下", "W41", 0, 0], ["线下", "W42", 2, 100]] + result = await ratio(zero) + assert result["reason_code"] == "zero_base" and result["reason"] == "基期的 orders 合计为 0，算不了比率" + orphan = [["线上", "W41", 5, 0], ["线上", "W42", 3, 100], ["线下", "W41", 1, 100], ["线下", "W42", 2, 100]] + assert (await ratio(orphan))["reason_code"] == "zero_base" + + +# -------------------------------------------------------------------------- +# factor +# -------------------------------------------------------------------------- + + +def _factor_snap(rows): + return snap("Q1", "SELECT week, SUM(amount) AS gmv, COUNT(*) AS orders, ROUND(AVG(amount), 2) AS aov FROM orders " + "GROUP BY week", ["week", "gmv", "orders", "aov"], rows) + + +async def factor(rows, **params): + params = {"query": "Q1", "period": "week", "base": "W41", "target": "W42", "product": "gmv", + "factors": ["orders", "aov"], "label": "销售额", **params} + return await analyze("decompose", "factor", snaps=[_factor_snap(rows)], **params) + + +def test_lmdi_contributions_add_up_to_the_change_within_float_error(): + rng = random.Random(3) + for _ in range(200): + k = rng.randint(2, 4) + base = [rng.uniform(0.5, 500) for _ in range(k)] + target = [b * rng.uniform(0.5, 1.8) for b in base] + m_b = math.prod(base) * (1 + rng.uniform(-1e-6, 1e-6)) # 取整过的因子：乘积差一点点 + m_t = math.prod(target) * (1 + rng.uniform(-1e-6, 1e-6)) + contrib = lmdi(base, target, m_b, m_t) + assert math.fsum(contrib) == pytest.approx(m_t - m_b, rel=1e-9, abs=1e-9 * max(m_b, m_t)) + same = lmdi([2.0, 3.0], [2.0, 3.0], 6.0, 6.0) + assert same == [0.0, 0.0] + + +async def test_factor_driver_with_rounded_factors(): + rows = [["W41", 1000, 137, "7.30"], ["W42", 1320, 160, "8.25"]] + result = await factor(rows) + assert result["verdict"]["code"] in ("driver", "both") + contrib = [Fraction(r[4]) for r in result["table"]["rows"]] + assert float(sum(contrib)) == pytest.approx(320, rel=1e-9) + clean = await factor([["W41", "1000", 100, "10"], ["W42", "1320", 120, "11"]]) + assert clean["verdict"] == {"code": "driver", "subjects": [["orders"]], "direction": "up"} + assert clean["phrase"].startswith("销售额上升 320（+32.0%），主要来自 orders 的变化（贡献 +") + assert clean["table"]["columns"] == ["factor", "base", "target", "pct", "contrib", "share"] + + +async def test_factor_call_errors_and_refusals(): + # 客单价写到两位小数：容差是它的半个末位（加上订单数的），差 1.5% 不是取整造成的 + error = await call_error("decompose", "factor", snaps=[_factor_snap([["W41", 1000, 100, "10.00"], + ["W42", 1300, 120, "11.00"]])], + query="Q1", period="week", base="W41", target="W42", product="gmv", + factors=["orders", "aov"]) + assert error.code == "not_product" and "本期差" in error.message + negative = await factor([["W41", 0, 0, 10], ["W42", 1320, 120, 11]]) + assert negative["reason_code"] == "non_positive" + flat = await factor([["W41", 1000, 100, 10], ["W42", 1000, 125, 8]]) + assert flat["verdict"]["code"] == "flat" + few = await call_error("decompose", "factor", snaps=[_factor_snap([["W41", 1, 1, 1], ["W42", 1, 1, 1]])], + query="Q1", period="week", base="W41", target="W42", product="gmv", factors=["orders"]) + assert few.code == "bad_factors" + + +async def test_ratio_headline_closure_and_drilling_into_a_channel(): + rows = [["线上", "W41", 50, 1000], ["线上", "W42", 80, 1000], ["线下", "W41", 20, 1000], ["线下", "W42", 26, 1000]] + head = snap("Q9", "SELECT week, SUM(refund_flag) * 1.0 / COUNT(*) AS rate FROM orders GROUP BY week", + ["week", "rate"], [["W41", "0.035"], ["W42", "0.053"]]) + params = {"query": "Q1", "by": "channel", "period": "week", "base": "W41", "target": "W42", + "numerator": "refunds", "denominator": "orders", "label": "退款率"} + closed = await analyze("decompose", "ratio", snaps=[_ratio_snap(rows), head], + headline={"base": "Q9.r0.rate", "target": "Q9.r1.rate"}, **params) + assert closed["verdict"]["code"] == "rate_driven" + off = snap("Q9", head["sql"], ["week", "rate"], [["W41", "0.035"], ["W42", "0.06"]]) + unclosed = await analyze("decompose", "ratio", snaps=[_ratio_snap(rows), off], + headline={"base": "Q9.r0.rate", "target": "Q9.r1.rate"}, **params) + assert unclosed["verdict"]["code"] == "unclosed" + assert unclosed["phrase"].startswith("按渠道拆开后各段合起来的比率和总数对不上") + # 线上按地区下钻：各地区的分子、分母之和要等于上一层线上那一段的 + region = snap("Q2", "SELECT region, week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders " + "WHERE channel = '线上' GROUP BY 1, 2", ["region", "week", "refunds", "orders"], + [["华东", "W41", 30, 600], ["华东", "W42", 55, 600], ["华北", "W41", 20, 400], ["华北", "W42", 25, 400]]) + child = await analyze("decompose", "ratio", snaps=[region], parents={"A1": closed}, + **{**params, "query": "Q2", "by": "region", "parent": {"of": "A1", "key": ["线上"]}}) + assert child["verdict"]["code"] != "unclosed" and child["params"]["depth"] == 1 + wrong = await call_error("decompose", "additive", snaps=[weekly()], parents={"A1": closed}, query="Q2", + by="region", period="week", base="2026-W41", target="2026-W42", measure="gmv", + parent={"of": "A1", "key": ["线上"]}) + assert wrong.code == "parent_not_decompose" + + +async def test_the_same_query_cannot_be_both_periods(): + error = await call_error("decompose", "additive", snaps=[weekly()], base_query="Q2", target_query="Q2", + by="region", measure="gmv") + assert error.code == "same_query" + error = await call_error("comparable", snaps=[weekly()], base_query="Q2", target_query="Q2") + assert error.code == "same_query" diff --git a/backend/tests/test_analysis_differ.py b/backend/tests/test_analysis_differ.py new file mode 100644 index 00000000..b8120d7b --- /dev/null +++ b/backend/tests/test_analysis_differ.py @@ -0,0 +1,138 @@ +"""差异检验（app/analysis/differ.py，细案 01 §5）：比率小样本走 Fisher、均值 Welch、Holm、rows 敏感性翻转。每个用例都顺带核对 +复算和运行逐字段相同。""" +from __future__ import annotations + +import math +from fractions import Fraction + +import pytest + +from analysis_samples import analyze, call_error, snap +from app.analysis import stats + +RATE_SQL = "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region" + + +def rates(rows): + return snap("Q1", RATE_SQL, ["region", "refunds", "orders"], rows) + + +async def rate(rows, **params): + params = {"query": "Q1", "by": "region", "numerator": "refunds", "denominator": "orders", "label": "退款率", **params} + return await analyze("differ", "rate", snaps=[rates(rows)], **params) + + +async def test_small_samples_use_fisher_and_say_it_may_be_noise(): + result = await rate([["西南", 2, 40], ["华东", 5, 230]]) + assert result["method"] == "differ.rate@1" + assert result["verdict"] == {"code": "no_evidence", "subjects": [["西南"], ["华东"]], "direction": None} + assert result["values"]["test"] == "fisher" and result["values"]["n_a"] == "40" and result["values"]["n_b"] == "230" + assert Fraction(result["values"]["p"]) == pytest.approx(stats.fisher_exact(2, 38, 5, 225), rel=1e-9) + assert result["phrase"] == ("西南和华东的退款率差异可能只是样本波动（样本量 40、230）；样本少（有格子的期望频数不到 5），" + "用了精确检验") + assert "p 值" not in result["phrase"] and "显著" not in result["phrase"] + assert result["settings"] == {"differ_alpha": 0.05, "min_expected": 5} + + +async def test_large_samples_use_the_pooled_z_test(): + result = await rate([["西南", 80, 1000], ["华东", 40, 1000]]) + assert result["verdict"] == {"code": "differs", "subjects": [["西南"], ["华东"]], "direction": "up"} + z, p = stats.pooled_z(80, 1000, 40, 1000) + assert Fraction(result["values"]["p"]) == pytest.approx(p, rel=1e-9) and result["values"]["test"] == "z" + low, high = stats.newcombe(80, 1000, 40, 1000) + assert Fraction(result["values"]["ci_lo"]) == pytest.approx(low, rel=1e-9) + assert result["values"]["a"] == "0.08" and result["values"]["diff"] == "0.04" + assert result["phrase"] == "西南的退款率（8.0%）高于华东（4.0%），差异站得住" + lower = await rate([["西南", 80, 1000], ["华东", 40, 1000]], pairs=[["华东", "西南"]]) + assert lower["verdict"]["direction"] == "down" and lower["phrase"].startswith("华东的退款率（4.0%）低于西南（8.0%）") + + +async def test_several_groups_each_against_the_rest_with_holm(): + rows = [["西南", 80, 1000], ["华东", 40, 1000], ["华南", 42, 1000], ["华北", 38, 1000]] + result = await rate(rows) + table = result["table"] + assert [r[0] for r in table["rows"]] == ["西南", "华东", "华南", "华北"] + assert table["rows"][0][1] == "其余各地区" + raw = [Fraction(r[10]) for r in table["rows"]] + adjusted = [Fraction(r[11]) for r in table["rows"]] + assert all(a >= p for a, p in zip(adjusted, raw)) + assert adjusted == pytest.approx(stats.holm([float(p) for p in raw]), rel=1e-9) + assert result["verdict"]["code"] == "differs" and result["verdict"]["subjects"] == [["西南"], ["其余各地区"]] + assert any(c["code"] == "multiple_comparisons" and c["text"] == "比了 4 对，已按次数校正" for c in result["caveats"]) + assert result["phrase"] == "比了 4 对，1 对差异站得住：西南的退款率（8.0%）高于其余各地区（4.0%）" + ref = await rate(rows, reference="华东") + assert [r[0] for r in ref["table"]["rows"]] == ["西南", "华南", "华北"] and ref["params"]["reference"] == "华东" + + +async def test_rate_call_errors(): + assert (await call_error("differ", "rate", snaps=[rates([["西南", 5, 4], ["华东", 1, 9]])], query="Q1", by="region", + numerator="refunds", denominator="orders")).code == "not_count" + assert (await call_error("differ", "rate", snaps=[rates([["西南", 1.5, 4], ["华东", 1, 9]])], query="Q1", + by="region", numerator="refunds", denominator="orders")).code == "not_count" + unknown = await call_error("differ", "rate", snaps=[rates([["西南", 1, 4], ["华东", 1, 9]])], query="Q1", + by="region", numerator="refunds", denominator="orders", pairs=[["西南", "东北"]]) + assert unknown.code == "unknown_group" and "西南、华东" in unknown.message + dup = await call_error("differ", "rate", snaps=[rates([["西南", 1, 4], ["西南", 1, 9]])], query="Q1", by="region", + numerator="refunds", denominator="orders") + assert dup.code == "duplicate_rows" + + +async def test_means_from_sums_and_squares_use_welch(): + sql = "SELECT channel, SUM(amount) AS s, SUM(amount * amount) AS ss, COUNT(*) AS n FROM orders GROUP BY channel" + rows = [["线上", 5000, 300000, 100], ["线下", 4000, 200000, 100]] + result = await analyze("differ", "mean", snaps=[snap("Q1", sql, ["channel", "s", "ss", "n"], rows)], query="Q1", + by="channel", sum="s", sumsq="ss", n="n", label="客单价") + va = (300000 - 5000 ** 2 / 100) / 99 + vb = (200000 - 4000 ** 2 / 100) / 99 + expected = stats.welch(50, va, 100, 40, vb, 100) + assert Fraction(result["values"]["p"]) == pytest.approx(expected.p, rel=1e-9) + assert result["verdict"]["code"] == "differs" and result["values"]["test"] == "welch" + assert result["phrase"] == "线上的客单价（50）高于线下（40），差异站得住" + assert result["method"] == "differ.welch@1" and result["settings"] == {"differ_alpha": 0.05, "min_n": 30} + + +async def test_means_from_mean_sd_and_n_and_small_samples(): + sql = "SELECT channel, avg_amount, sd_amount, n FROM channel_stats" + rows = [["线上", 52, 12, 12], ["线下", 50, 11, 400]] + result = await analyze("differ", "mean", snaps=[snap("Q1", sql, ["channel", "avg_amount", "sd_amount", "n"], rows)], + query="Q1", by="channel", mean="avg_amount", sd="sd_amount", n="n", label="客单价") + assert result["verdict"]["code"] == "no_evidence" + note = next(c for c in result["caveats"] if c["code"] == "small_sample") + assert note["text"] == "有一组的样本数不到 30（线上 12），结论不稳" + assert result["phrase"].startswith("线上和线下的客单价差异可能只是样本波动（样本量 12、400）") + + +async def test_rows_mode_flags_a_single_value_that_flips_the_verdict(): + base = [["线下", v] for v in [40, 60] * 200] + online = [["线上", f"{v:.1f}"] for v in [41.3, 61.3] * 200] + online[-1] = ["线上", "231.3"] + s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], online + base) + result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount", label="客单价") + assert result["method"] == "differ.welch_rows@1" + assert result["verdict"]["code"] == "differs" + note = next(c for c in result["caveats"] if c["code"] == "outlier_sensitive") + assert note["text"] == "去掉最大的一笔后，差异不再站得住" + assert note["text"] in result["phrase"] + hidden = [["线上", 50 + (i % 7)] for i in range(30)] + [["线下", 48 + (i % 5)] for i in range(30)] + [["线上", 400]] + s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], hidden) + flipped = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount", label="客单价") + assert flipped["verdict"]["code"] == "no_evidence" + assert any(c["text"] == "去掉最大的一笔后，差异变得站得住" for c in flipped["caveats"]) + + +async def test_two_periods_of_the_same_group(): + sql = "SELECT week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders WHERE region = '西南' GROUP BY week" + s = snap("Q1", sql, ["week", "refunds", "orders"], [["W41", 30, 1000], ["W42", 60, 1000], ["W40", 31, 1000]]) + result = await analyze("differ", "rate", snaps=[s], query="Q1", period="week", base="W41", target="W42", + numerator="refunds", denominator="orders", label="退款率") + assert result["verdict"] == {"code": "differs", "subjects": [["W42"], ["W41"]], "direction": "up"} + assert result["inputs"][0]["rows"] == [0, 1] + assert result["phrase"] == "W42 的退款率（6.0%）高于 W41（3.0%），差异站得住" + + +async def test_a_group_with_one_value_cannot_be_tested(): + s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], [["线上", 5], ["线下", 4], ["线下", 6]]) + result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount") + assert result["verdict"]["code"] == "no_evidence" and result["values"]["p"] is None + assert result["table"]["rows"][0][-1] == "样本不够，算不了" + assert math.isfinite(float(result["values"]["diff"])) diff --git a/backend/tests/test_analysis_probe.py b/backend/tests/test_analysis_probe.py new file mode 100644 index 00000000..f84a873e --- /dev/null +++ b/backend/tests/test_analysis_probe.py @@ -0,0 +1,129 @@ +"""可比性检查的现查（app/analysis/probe.py）：make_prober 在真的数据源上探表的日期范围、在库里算按运行当天现算的界，遮罩列 +不探、超时限不探、现查不成退回剖析事实；Book 先登记、查完再算，结论只记最后真用到的。库是现造的 SQLite 文件，表名、列名、 +数字都是虚构的。""" +from __future__ import annotations + +import sqlite3 +import uuid +from datetime import date, datetime, timedelta, timezone + +import pytest + +from analysis_samples import CATALOGS, recomputed, snap +from app import analysis +from app.analysis.contract import effective_settings, same +from app.analysis.probe import Book, make_prober +from app.data import introspect +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource + +CLOCK = "2026-10-21T09:30:00" +#: SQLite 的 date('now') 按 UTC 算：表里的数据造到 UTC 的昨天 +UTC_TODAY = datetime.now(timezone.utc).date() +LAST_DAY = UTC_TODAY - timedelta(days=1) + + +@pytest.fixture +async def shop(tmp_path): + """orders：order_date 从 2026-01-01 到昨天（UTC），每天一单；customers 的 signup_date、phone 是遮罩列。""" + path = tmp_path / f"{uuid.uuid4().hex[:8]}.db" + db = sqlite3.connect(path) + db.executescript("CREATE TABLE orders (order_date TEXT, region TEXT, amount INTEGER);" + "CREATE TABLE customers (signup_date TEXT, phone TEXT);" + "INSERT INTO customers VALUES ('2026-01-05', '13900000000');") + day = date(2026, 1, 1) + while day <= LAST_DAY: + db.execute("INSERT INTO orders VALUES (?, '华东', 100)", (day.isoformat(),)) + day += timedelta(days=1) + db.commit() + db.close() + name = f"shop_{uuid.uuid4().hex[:8]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True, + options={"mask_columns": ["phone", "signup_date"]}) + session.add(row) + await session.flush() + row.schema_cache = await introspect.introspect(row) + await session.commit() + await session.refresh(row) + yield row + await engines.invalidate(row.id) + + +async def test_the_live_range_of_a_column(shop): + prober = make_prober({shop.name: shop}, clock=lambda: CLOCK) + probe = await prober(shop.name, "orders", "order_date") + assert probe == {"source": shop.name, "table": "orders", "column": "order_date", "min": "2026-01-01", + "max": LAST_DAY.isoformat(), "basis": "live", "at": CLOCK} + assert prober.dialect(shop.name) == "sqlite" and prober.dialect("nope") is None + assert await prober("nope", "orders", "order_date") is None + + +async def test_masked_columns_are_never_probed(shop): + prober = make_prober({shop.name: shop}) + assert await prober(shop.name, "customers", "signup_date") is None # 数据源设的遮罩列 + explicit = make_prober({shop.name: shop}, masked={shop.name: ["ORDER_DATE"]}) + assert await explicit(shop.name, "orders", "order_date") is None + + +async def test_a_failed_probe_falls_back_to_profile_facts(shop): + async def facts(table): + return {"at": "2026-10-20T02:00:00", "date_ranges": [{"column": "paid_on", "min": "2026-02-01", + "max": "2026-10-19"}]} + + prober = make_prober({shop.name: shop}, facts={shop.name: facts}, clock=lambda: CLOCK) + probe = await prober(shop.name, "payments", "paid_on") # 没有这张表：现查不成 + assert probe == {"source": shop.name, "table": "payments", "column": "paid_on", "min": "2026-02-01", + "max": "2026-10-19", "basis": "profile", "at": "2026-10-20"} + + +async def test_the_budget_stops_further_probes(shop): + prober = make_prober({shop.name: shop}, budget=0) + await prober(shop.name, "orders", "order_date") + assert await prober(shop.name, "orders", "order_date") is None + + +async def test_bounds_are_computed_in_the_database(shop): + prober = make_prober({shop.name: shop}, clock=lambda: CLOCK) + probe = await prober.bounds(shop.name, "DATE('now', '-7 days')") + assert probe["expr"] == "DATE('now', '-7 days')" and probe["basis"] == "live" and probe["at"] == CLOCK + assert date.fromisoformat(probe["value"]) == UTC_TODAY - timedelta(days=7) + assert await prober.bounds(shop.name, "no_such_function(1)") is None + + +def test_the_book_registers_first_and_records_only_what_was_used(): + book = Book({}) + assert book.range("shop", "orders", "order_date") is None and book.bound("shop", "DATE('now')") is None + assert book.missing == [("range", "shop", "orders", "order_date"), ("bound", "shop", "DATE('now')")] + found = {"source": "shop", "table": "orders", "column": "order_date", "min": "2026-01-01", "max": "2026-10-18", + "basis": "live", "at": CLOCK} + filled = Book({("range", "shop", "orders", "order_date"): found, ("bound", "shop", "DATE('now')"): None}) + assert filled.range("shop", "ORDERS", "Order_Date") is found # 表名、列名不分大小写 + filled.bound("shop", "DATE('now')") + assert filled.missing == [] and filled.used == [found] + replay = Book([found]) # 复算：记下的就是全部 + assert replay.bound("shop", "DATE('now', '-1 day')") is None and replay.missing == [] + + +async def test_a_full_comparability_check_against_the_database_records_its_probes(shop): + sql = ("SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', '{a}') " + "AND order_date < date('now', '{b}')") + base = snap("Q1", sql.format(a="-14 days", b="-7 days"), ["gmv"], [[700]], source=shop.name) + target = snap("Q2", sql.format(a="-7 days", b="0 days"), ["gmv"], [[300]], source=shop.name) + prober = make_prober({shop.name: shop}, clock=lambda: CLOCK) + result = await analysis.run({"op": "comparable", "params": {"base_query": "Q1", "target_query": "Q2", + "measure": "gmv"}}, + {"Q1": base, "Q2": target}.get, settings=effective_settings(None), + context={"today": UTC_TODAY, "probe": prober, + "catalogs": {shop.name: CATALOGS["shop"]}}) + exprs = [p["expr"] for p in result["probes"] if "expr" in p] + assert exprs == ["DATE('now', '-14 days')", "DATE('now', '-7 days')", "DATE('now', '0 days')"] + ranges = [p for p in result["probes"] if "table" in p] + assert [(p["table"], p["column"], p["max"], p["basis"]) for p in ranges] == \ + [("orders", "order_date", LAST_DAY.isoformat(), "live")] + assert result["values"]["base_days"] == "7" and result["values"]["target_days"] == "7" + assert result["values"]["target_end"] == LAST_DAY.isoformat() + assert all(i["dialect"] == "sqlite" for i in result["inputs"]) + assert result["verdict"]["code"] == "comparable" and result["phrase"] == "两期可比" + assert same(result, recomputed(result, [base, target])) diff --git a/backend/tests/test_analysis_run.py b/backend/tests/test_analysis_run.py new file mode 100644 index 00000000..262716ca --- /dev/null +++ b/backend/tests/test_analysis_run.py @@ -0,0 +1,152 @@ +"""analysis.run / recompute（app/analysis/__init__.py）：结论的形状严格按契约、同输入两次 run 逐字节相同、复算和运行逐字段相同且 +只用记下的 probes（不现查、不读目录）、调用方写的系统字段不收、复算取不回输入时的说法。""" +from __future__ import annotations + +import json +from datetime import date, timedelta + +import pytest + +from analysis_samples import CATALOGS, analyze, call_error, fixed_prober, recomputed, snap +from app import analysis +from app.analysis import contract as c +from app.analysis.contract import CallError, effective_settings, same + +WEEKLY_SQL = "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week" +WEEKLY = snap("Q2", WEEKLY_SQL, ["region", "week", "gmv"], + [["华东", "2026-W41", 820], ["华东", "2026-W42", 700], ["华北", "2026-W41", 600], + ["华北", "2026-W42", 560], ["华南", "2026-W41", 700], ["华南", "2026-W42", 666]]) +SERIES = snap("Q1", "SELECT DATE(order_date, 'weekday 0', '-6 days') AS week, SUM(amount) AS gmv FROM orders GROUP BY 1", + ["week", "gmv"], [[(date(2026, 6, 1) + timedelta(days=7 * i)).isoformat(), 100 + (i % 4) * 3 + i] + for i in range(20)]) +RATES = snap("Q3", "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region", + ["region", "refunds", "orders"], [["西南", 80, 1000], ["华东", 40, 1000], ["华北", 41, 1000]]) +BASE = snap("Q4", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' AND " + "order_date < '2026-10-12' GROUP BY region", ["region", "gmv"], [["华东", 820], ["华北", 600]]) +TARGET = snap("Q5", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-12' AND " + "order_date < '2026-10-19' GROUP BY region", ["region", "gmv"], [["华东", 700], ["华北", 590]]) +SNAPS = [WEEKLY, SERIES, RATES, BASE, TARGET] +PROBE = {("orders", "order_date"): ("2025-01-01", "2026-10-20")} + +CALLS = [ + ("decompose", "additive", {"query": "Q2", "by": "region", "period": "week", "base": "2026-W41", + "target": "2026-W42", "measure": "gmv", "label": "销售额"}), + ("decompose", "additive", {"base_query": "Q4", "target_query": "Q5", "by": "region", "measure": "gmv"}), + ("baseline", None, {"query": "Q1", "period": "week", "measure": "gmv"}), + ("trend", None, {"query": "Q1", "period": "week", "measure": "gmv"}), + ("differ", "rate", {"query": "Q3", "by": "region", "numerator": "refunds", "denominator": "orders"}), + ("comparable", None, {"base_query": "Q4", "target_query": "Q5", "measure": "gmv"}), +] + + +async def _run(op, mode, params, *, probe=None, **kw): + return await analyze(op, mode, snaps=SNAPS, probe=probe or fixed_prober(PROBE), check=False, **params, **kw) + + +def _assert_shape(result): + """结论的字段、取值严格按契约。""" + assert set(result) == set(c.AnalysisResult.__annotations__) + assert result["op"] in c.OPS and result["method"] == c.METHODS[(result["op"], result["mode"])] + assert result["status"] in ("ok", "refused") + if result["status"] == "refused": + assert result["reason_code"] in c.REASONS and result["reason"] and result["verdict"] is None + else: + assert result["reason_code"] is None and result["reason"] is None + assert result["verdict"]["code"] in c.VERDICTS[result["op"]] + assert set(result["verdict"]) == {"code", "subjects", "direction"} + assert all(isinstance(s, list) and all(isinstance(x, str) for x in s) for s in result["verdict"]["subjects"]) + assert result["verdict"]["direction"] in (None, "up", "down") + table = result["table"] + assert set(table) == {"columns", "labels", "rows"} and set(table["labels"]) == set(table["columns"]) + assert all(len(r) == len(table["columns"]) for r in table["rows"]) + assert all(v is None or isinstance(v, str) for v in result["values"].values()) + assert all(set(cv) == {"code", "text"} and cv["code"] in c.CAVEATS for cv in result["caveats"]) + assert set(result["settings"]) <= set(c.DEFAULTS) + assert all(i.get("artifact") for i in result["inputs"]) + assert result["criterion"] and result["phrase"] + json.dumps(result, ensure_ascii=False) # 写得进 JSON + + +@pytest.mark.parametrize("op,mode,params", CALLS) +async def test_every_result_has_the_contract_shape_and_recomputes_the_same(op, mode, params): + result = await _run(op, mode, params) + _assert_shape(result) + again = recomputed(result, SNAPS) + assert same(result, again) + assert json.dumps(again, sort_keys=True, ensure_ascii=False) == json.dumps(result, sort_keys=True, ensure_ascii=False) + + +@pytest.mark.parametrize("op,mode,params", CALLS) +async def test_the_same_inputs_give_byte_identical_results(op, mode, params): + one = await _run(op, mode, params) + two = await _run(op, mode, params) + assert json.dumps(one, sort_keys=True, ensure_ascii=False) == json.dumps(two, sort_keys=True, ensure_ascii=False) + + +async def test_recompute_uses_the_recorded_probes_and_facts_not_live_ones(): + result = await _run("comparable", None, {"base_query": "Q4", "target_query": "Q5", "measure": "gmv"}) + assert result["verdict"]["code"] == "comparable" and len(result["probes"]) == 1 + # 记下的范围换一个（表里数据只到 10-16）：复算按记下的算，结论跟着变——它没有现查 + edited = {**result, "probes": [{**result["probes"][0], "max": "2026-10-16"}]} + assert recomputed(edited, SNAPS)["verdict"]["code"] == "incomparable" + # 目录事后改了也不影响复算：目录事实记在 params 里 + decompose = await _run(*CALLS[0]) + assert decompose["params"]["catalog"]["labels"]["region"] == "地区" + CATALOGS["shop"]["orders"]["columns"]["region"]["label"]["value"] = "大区" + try: + assert same(decompose, recomputed(decompose, SNAPS)) + finally: + CATALOGS["shop"]["orders"]["columns"]["region"]["label"]["value"] = "地区" + + +async def test_system_params_written_by_the_caller_are_ignored(): + params = {"query": "Q1", "period": "week", "measure": "gmv", "today": "2020-01-01", + "catalog": {"labels": {"gmv": "假的叫法"}}, "depth": 9} + result = await analysis.run({"op": "baseline", "params": params}, {s["ref"]: s for s in SNAPS}.get, + settings=effective_settings(None), + context={"today": date(2026, 10, 21), "catalogs": CATALOGS}) + assert result["params"]["today"] == "2026-10-21" + assert "假的叫法" not in result["phrase"] and "depth" not in result["params"] + + +async def test_analysis_card_overrides_reach_the_criterion_and_settings(): + params = CALLS[0][2] + result = await _run("decompose", "additive", params, settings={"main_share": 0.6}) + assert result["settings"]["main_share"] == 0.6 and "占总变化的 60% 以上" in result["criterion"] + + +async def test_call_errors_for_unknown_ops_and_modes(): + with pytest.raises(CallError) as e: + await analysis.run({"op": "forecast", "params": {}}, {}.get, settings=effective_settings(None), context={}) + assert e.value.code == "unknown_op" and "变化拆解" in e.value.message + with pytest.raises(CallError) as e: + await analysis.run({"op": "decompose", "mode": "share", "params": {}}, {}.get, + settings=effective_settings(None), context={}) + assert e.value.code == "unknown_mode" and "additive / ratio / factor" in e.value.message + missing = await call_error("baseline", snaps=SNAPS, query="Q1", period="week") + assert missing.code == "missing_param" and missing.message == "没写度量列" + + +async def test_recompute_says_when_an_input_cannot_be_loaded(): + result = await _run(*CALLS[0]) + with pytest.raises(CallError) as e: + analysis.recompute(result, lambda artifact: None) + assert e.value.code == "input_missing" and e.value.message == "复算取不回 Q2 的查询快照" + + +async def test_the_hypothesis_is_recorded_but_never_rendered(): + result = await _run(*CALLS[0], hypothesis="可能是华东促销结束 30% 的影响") + assert result["hypothesis"] == "可能是华东促销结束 30% 的影响" + assert "促销" not in result["phrase"] and "30" not in result["phrase"] + assert same(result, {**recomputed(result, SNAPS), "hypothesis": None}) + + +async def test_ledger_entries_name_the_real_input_snapshots(): + result = await _run(*CALLS[1]) + entry = c.ledger_entry(result, node_id="agent", exec_no=1, artifact="f" * 64, call_id="call-1") + assert entry["inputs"] == [BASE["artifact"], TARGET["artifact"]] and entry["op"] == "decompose" + + +def test_the_call_limit_error_is_spoken_with_the_limit(): + error = analysis.exhausted({"max_calls": 5}) + assert error.code == "max_calls" and error.message == "这次运行的分析次数已经用完（最多 5 次）：用已有的分析结论写，不再新做分析" diff --git a/backend/tests/test_analysis_series.py b/backend/tests/test_analysis_series.py new file mode 100644 index 00000000..bb4c4774 --- /dev/null +++ b/backend/tests/test_analysis_series.py @@ -0,0 +1,210 @@ +"""波动基线、趋势（app/analysis/series.py，细案 01 §3、§4）：常规 / 偏离 / 明显偏离、σ̂ = 0、历史不够、本期不完整、缺期、日数据 +lag 7、新高新低；上升 / 下降 / 无趋势、连续期数、缺期。每个用例都顺带核对复算和运行逐字段相同。""" +from __future__ import annotations + +from datetime import date, timedelta +from fractions import Fraction + +import pytest + +from analysis_samples import analyze, call_error, fixed_prober, snap +from app.analysis.series import MAD_SCALE, run_length + +SQL = "SELECT DATE(order_date, 'weekday 0', '-6 days') AS week, COUNT(*) AS orders FROM orders GROUP BY 1" +#: 每周一的日期：最后一周是 2026-10-12（到 10-18，运行当天 10-21 之前已经过完） +LAST_MONDAY = date(2026, 10, 12) + + +def weeks(values, *, last=LAST_MONDAY): + start = last - timedelta(days=7 * (len(values) - 1)) + return [[(start + timedelta(days=7 * i)).isoformat(), v] for i, v in enumerate(values)] + + +def series(rows, sql=SQL, cols=("week", "orders")): + return snap("Q1", sql, list(cols), rows, types={cols[0]: "text", cols[1]: "number"}) + + +async def baseline(rows, **params): + params = {"query": "Q1", "period": "week", "measure": "orders", "label": "订单数", **params} + return await analyze("baseline", snaps=[series(rows)], **params) + + +async def trend(rows, **params): + params = {"query": "Q1", "period": "week", "measure": "orders", "label": "线上销售额", **params} + return await analyze("trend", snaps=[series(rows)], **params) + + +NOISE = [100, 102, 98, 101, 99, 103, 100, 97, 102, 101, 99, 100, 98, 103, 101, 100, 99, 102, 100] + + +# -------------------------------------------------------------------------- +# 波动基线 +# -------------------------------------------------------------------------- + + +async def test_level_basis_with_hand_computed_median_and_mad(): + rows = [[i + 1, v] for i, v in enumerate([10, 12, 11, 13, 9, 10, 12, 11, 16])] + result = await baseline(rows, basis="level") + # 历史中位数 11，MAD 1，σ̂ = 1.4826，z = (16 − 11) / 1.4826 + values = result["values"] + assert values["median"] == "11" and values["n_history"] == "8" and values["rank"] == "1" + assert Fraction(values["z"]) == pytest.approx(Fraction(5) / MAD_SCALE) + assert values["band_lo"] == "8.0348" and values["band_hi"] == "13.9652" + assert result["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "up"} + assert result["phrase"].startswith("订单数本期为 16，明显超出过去 8 期的常规波动范围（8.03 至 13.97）") + assert any(c["code"] == "record_high" for c in result["caveats"]) + assert result["params"]["grain"] == "seq" and result["params"]["lag"] == 1 + assert result["settings"] == {"k_warn": 2.0, "k_alert": 3.0, "min_history": 8, "max_history": 52} + assert result["table"]["rows"][-1][-1] == "本期" + + +async def test_change_basis_within_outside_and_far_outside(): + probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")}) + within = await analyze("baseline", snaps=[series(weeks(NOISE + [101]))], probe=probe, query="Q1", period="week", + measure="orders", label="订单数") + assert within["verdict"]["code"] == "within_band" + assert within["phrase"].startswith("订单数本期变化 +1.0%，在过去 18 周的常规波动范围内（") + assert within["params"]["today"] == "2026-10-21" and within["params"]["grain"] == "week" + assert within["probes"][0]["table"] == "orders" and within["probes"][0]["column"] == "order_date" + assert not any(c["code"] == "comparability" for c in within["caveats"]) + outside = await analyze("baseline", snaps=[series(weeks(NOISE + [107]))], probe=probe, query="Q1", + period="week", measure="orders") + assert outside["verdict"]["code"] == "outside_band" and "超出过去 18 周" in outside["phrase"] + far = await analyze("baseline", snaps=[series(weeks(NOISE + [70]))], probe=probe, query="Q1", period="week", + measure="orders", label="订单数") + assert far["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "down"} + assert far["phrase"].startswith("订单数本期变化 -30.0%，明显超出过去 18 周的常规波动范围") + assert any(c["code"] == "record_low" and c["text"] == "本期的订单数是过去 18 周里最低的" for c in far["caveats"]) + + +async def test_without_a_table_range_completeness_is_said_to_be_unverified_but_not_in_the_phrase(): + result = await baseline(weeks(NOISE + [101])) + note = next(c for c in result["caveats"] if c["code"] == "comparability") + assert note["text"] == "没能核对本期的数据是否到齐（查不到表里这一列的日期范围）" + assert note["text"] not in result["phrase"] + + +async def test_flat_history_has_no_spread(): + rows = [[i + 1, 50] for i in range(10)] + same = await baseline(rows, basis="level") + assert same["verdict"]["code"] == "within_band" and same["values"]["z"] is None + assert same["phrase"] == "订单数本期为 50，和过去 9 期一样（每期都是 50）" + jump = await baseline(rows[:-1] + [[10, 51]], basis="level") + assert jump["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "up"} + assert jump["phrase"].startswith("订单数本期为 51，明显超出过去 9 期的常规波动范围（每期都是 50）") + + +async def test_short_history_partial_target_gaps_and_zero_bases_are_refused(): + short = await baseline(weeks([100, 101, 99, 102, 98, 100])) + # 6 周：第一周没有上一周可比，本期不算历史，历史的变化只有 4 个 + assert short["reason_code"] == "history_short" and short["reason"] == "只有 4 期历史，至少要 8 期" + assert short["phrase"] == "波动基线没有做成：只有 4 期历史，至少要 8 期" + # 最后一周是 10-19 到 10-25：运行当天 10-21 还没过完 + partial = await baseline(weeks(NOISE + [60], last=date(2026, 10, 19))) + assert partial["reason_code"] == "partial_target" + assert partial["reason"] == "本期数据只到 2026-10-20，还差 5 天，和完整的周比没有意义" + # 周过完了，但表里的数据只到 10-17 + probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-17")}) + late = await analyze("baseline", snaps=[series(weeks(NOISE + [60]))], probe=probe, query="Q1", period="week", + measure="orders") + assert late["reason"] == "本期数据只到 2026-10-17，还差 1 天，和完整的周比没有意义" + gap_rows = weeks(NOISE + [100, 100]) + del gap_rows[-2] # 本期的上一周没有 + gap = await baseline(gap_rows) + assert gap["reason_code"] == "target_gap" + zero = await baseline(weeks(NOISE + [0, 10])) + assert zero["reason_code"] == "zero_base" + unreadable = await baseline([["上周", 1], ["本周", 2]]) + assert unreadable["reason_code"] == "period_unreadable" and "上周" in unreadable["reason"] + + +async def test_daily_series_compare_with_the_same_weekday_last_week(): + start = date(2026, 7, 1) + rows = [[(start + timedelta(days=i)).isoformat(), 100 + (i % 7) * 10] for i in range(100)] + result = await analyze("baseline", snaps=[series(rows, cols=("day", "orders"))], query="Q1", period="day", + measure="orders", label="订单数", check=True) + assert result["params"]["grain"] == "day" and result["params"]["lag"] == 7 + # 星期效应：和上周同一天比，每天的变化都是 0 + assert result["values"]["median"] == "0" and result["verdict"]["code"] == "within_band" + assert result["phrase"].startswith("订单数比上周同一天变化 0.0%，和过去 52 天一样") + yoy = await baseline(weeks(NOISE * 3), lag=52) # 57 周：能和 52 周前比的历史只有 4 期 + assert yoy["reason_code"] == "history_short" and yoy["reason"] == "只有 4 期历史，至少要 8 期" + assert (await baseline(weeks(NOISE * 4), lag=52))["status"] == "ok" + assert (await call_error("baseline", snaps=[series(weeks(NOISE))], query="Q1", period="week", measure="orders", + lag=0)).code == "bad_lag" + + +async def test_duplicate_periods_and_given_targets(): + rows = weeks(NOISE + [101]) + error = await call_error("baseline", snaps=[series(rows + [rows[-1]])], query="Q1", period="week", + measure="orders") + assert error.code == "duplicate_rows" and "在 SQL 里按 week 汇总" in error.message + earlier = await baseline(rows, target=rows[-3][0]) + assert earlier["params"]["target"] == rows[-3][0] and earlier["values"]["n_history"] == "16" + missing = await call_error("baseline", snaps=[series(rows)], query="Q1", period="week", measure="orders", + target="2020-01-06") + assert missing.code == "period_not_found" + + +async def test_iso_week_labels_and_months(): + labels = [[f"2026-W{w:02d}", v] for w, v in zip(range(22, 42), NOISE + [101])] + result = await baseline(labels) + assert result["params"]["grain"] == "week" and result["verdict"]["code"] == "within_band" + months = [[f"2025-{m:02d}", 100 + m] for m in range(1, 13)] + [[f"2026-{m:02d}", 112 + m] for m in range(1, 10)] + monthly = await baseline(months) + assert monthly["params"]["grain"] == "month" and "过去 19 个月" in monthly["phrase"] + + +# -------------------------------------------------------------------------- +# 趋势 +# -------------------------------------------------------------------------- + + +async def test_rising_and_falling_trends_with_run_length(): + rising = await trend(weeks([100 + 3 * i + (i % 3) for i in range(12)])) + assert rising["verdict"] == {"code": "rising", "subjects": [], "direction": "up"} + assert rising["values"]["n"] == "12" and rising["values"]["run_length"] == "11" + assert rising["phrase"].startswith("过去 12 周线上销售额呈上升趋势（每周约 +") + assert "，已连续 11 周上升" in rising["phrase"] + assert rising["settings"] == {"trend_min_points": 6, "trend_alpha": 0.05, "max_history": 52} + falling = await trend(weeks([200, 196, 197, 190, 185, 186, 180, 181, 175, 170])) + assert falling["verdict"]["code"] == "falling" + assert falling["values"]["run_length"] == "2" and "已连续" not in falling["phrase"] + + +async def test_no_trend_and_exact_small_sample_p(): + flat = await trend(weeks([100, 104, 98, 103, 97, 101, 99, 102])) + assert flat["verdict"] == {"code": "no_trend", "subjects": [], "direction": None} + assert flat["phrase"] == "过去 8 周线上销售额没有明显的上升或下降趋势" + six = await trend(weeks([1, 2, 3, 4, 5, 6])) + # 没有并列、n ≤ 10：精确分布，单调的 6 期双侧 p = 2 / 6! + assert Fraction(six["values"]["p"]) == pytest.approx(Fraction(2, 720)) + assert six["verdict"]["code"] == "rising" and six["values"]["slope"] == "1" + + +def test_run_length_counts_consecutive_changes_from_the_end(): + assert run_length([Fraction(v) for v in (1, 2, 3, 2, 3, 4, 5)]) == (3, "up") + assert run_length([Fraction(v) for v in (5, 4, 3)]) == (2, "down") + assert run_length([Fraction(v) for v in (1, 2, 2)]) == (0, None) + + +async def test_trend_needs_enough_consecutive_points(): + rows = weeks([100 + i for i in range(10)]) + del rows[-4] # 最后连着的只有 3 期 + gap = await trend(rows) + assert gap["reason_code"] == "series_gap" and gap["reason"] == "最后连着的只有 3 期，再往前缺期（至少要连着的 6 期）" + short = await trend(weeks([1, 2, 3, 4])) + assert short["reason_code"] == "history_short" and short["reason"] == "只有 4 期，至少要 6 期" + partial = await trend(weeks([100 + i for i in range(10)], last=date(2026, 10, 19))) + assert partial["reason_code"] == "partial_target" + + +async def test_daily_trends_warn_about_weekday_effects(): + start = date(2026, 8, 1) + rows = [[(start + timedelta(days=i)).isoformat(), 100 + i] for i in range(30)] + result = await analyze("trend", snaps=[series(rows, cols=("day", "orders"))], query="Q1", period="day", + measure="orders") + assert result["verdict"]["code"] == "rising" + assert any(c["code"] == "weekday_effect" and c["text"] == "日数据有星期效应，看趋势建议按周汇总" + for c in result["caveats"]) + assert "日数据有星期效应" in result["phrase"]