From 9af50fb47480dec7e625cfec9609148e3acbe352 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 13:03:12 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E5=88=86=E6=9E=90=E7=AE=97=E5=AD=90?= =?UTF-8?q?=E7=9A=84=E7=BB=9F=E8=AE=A1=E5=87=BD=E6=95=B0=EF=BC=88app/analy?= =?UTF-8?q?sis/stats.py=EF=BC=8C=E7=BA=AF=20Python=E3=80=81=E4=B8=8D?= =?UTF-8?q?=E5=8A=A0=E4=BE=9D=E8=B5=96=EF=BC=89=EF=BC=9A=E6=AD=A3=E6=80=81?= =?UTF-8?q?=E5=88=86=E5=B8=83=E5=87=BD=E6=95=B0=EF=BC=88math.erfc=EF=BC=89?= =?UTF-8?q?=E4=B8=8E=E5=88=86=E4=BD=8D=E6=95=B0=EF=BC=88Acklam=20=E9=80=BC?= =?UTF-8?q?=E8=BF=91=20+=20Halley=20=E4=BF=AE=E6=AD=A3=EF=BC=89=E3=80=81t?= =?UTF-8?q?=20=E5=88=86=E5=B8=83=EF=BC=88=E6=AD=A3=E5=88=99=E5=8C=96?= =?UTF-8?q?=E4=B8=8D=E5=AE=8C=E5=85=A8=20=CE=B2=20=E7=9A=84=E8=BF=9E?= =?UTF-8?q?=E5=88=86=E5=BC=8F=EF=BC=8C=E8=87=AA=E7=94=B1=E5=BA=A6=E5=BE=88?= =?UTF-8?q?=E5=A4=A7=E6=97=B6=E6=8C=89=E6=AD=A3=E6=80=81=EF=BC=89=E4=B8=8E?= =?UTF-8?q?=E5=88=86=E4=BD=8D=E6=95=B0=EF=BC=88=E4=BA=8C=E5=88=86=EF=BC=89?= =?UTF-8?q?=E3=80=81Wilson=EF=BC=88=E4=BB=8E=20eval/stats=20=E7=A7=BB?= =?UTF-8?q?=E8=BF=87=E6=9D=A5=EF=BC=8C=E8=AF=84=E6=B5=8B=E6=94=B9=E4=B8=BA?= =?UTF-8?q?=E5=BC=95=E7=94=A8=E3=80=81=E8=A1=8C=E4=B8=BA=E4=B8=8D=E5=8F=98?= =?UTF-8?q?=EF=BC=89=E3=80=81Newcombe=20=E5=8C=BA=E9=97=B4=E3=80=81?= =?UTF-8?q?=E5=90=88=E5=B9=B6=E6=AF=94=E4=BE=8B=20z=20=E6=A3=80=E9=AA=8C?= =?UTF-8?q?=E3=80=81Fisher=20=E7=B2=BE=E7=A1=AE=E6=A3=80=E9=AA=8C=EF=BC=88?= =?UTF-8?q?lgamma=EF=BC=89=E3=80=81Welch=20t=20=E6=A3=80=E9=AA=8C=E4=B8=8E?= =?UTF-8?q?=E5=8C=BA=E9=97=B4=E3=80=81Holm=20=E6=A0=A1=E6=AD=A3=E3=80=81Ma?= =?UTF-8?q?nn=E2=80=93Kendall=EF=BC=88n=20=E2=89=A4=2010=20=E4=B8=94?= =?UTF-8?q?=E6=97=A0=E5=B9=B6=E5=88=97=E7=94=A8=E9=80=86=E5=BA=8F=E6=95=B0?= =?UTF-8?q?=E9=80=92=E6=8E=A8=E7=9A=84=E7=B2=BE=E7=A1=AE=E5=88=86=E5=B8=83?= =?UTF-8?q?=EF=BC=8C=E5=90=A6=E5=88=99=E6=AD=A3=E6=80=81=E8=BF=91=E4=BC=BC?= =?UTF-8?q?=E5=B8=A6=E5=B9=B6=E5=88=97=E4=BF=AE=E6=AD=A3=E5=92=8C=E8=BF=9E?= =?UTF-8?q?=E7=BB=AD=E6=80=A7=E6=A0=A1=E6=AD=A3=EF=BC=89=E3=80=81Theil?= =?UTF-8?q?=E2=80=93Sen=EF=BC=88=E7=B2=BE=E7=A1=AE=E7=9A=84=E6=95=B0?= =?UTF-8?q?=E5=9C=A8=E5=85=AC=E5=88=86=E6=AF=8D=E4=B8=8A=E6=8C=89=E6=95=B4?= =?UTF-8?q?=E6=95=B0=E6=AF=94=EF=BC=8C=E5=8F=AA=E5=AF=B9=E6=B5=AE=E7=82=B9?= =?UTF-8?q?=E7=9B=B8=E7=AD=89=E7=9A=84=E9=82=A3=E5=87=A0=E4=B8=AA=E7=B2=BE?= =?UTF-8?q?=E7=A1=AE=E6=AF=94=EF=BC=89=E3=80=81=E4=B8=AD=E4=BD=8D=E6=95=B0?= =?UTF-8?q?=20/=20MAD=20/=20=E5=90=8D=E6=AC=A1=EF=BC=9B=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=94=A8=E5=85=AC=E8=AE=A4=E8=A1=A8=E5=80=BC=E9=92=89=E4=BD=8F?= =?UTF-8?q?=EF=BC=88=E6=AD=A3=E6=80=81=200.975=20=E5=88=86=E4=BD=8D?= =?UTF-8?q?=E3=80=81t(10)=20=E4=B8=B4=E7=95=8C=E5=80=BC=E3=80=81=E5=93=81?= =?UTF-8?q?=E8=8C=B6=E8=A1=A8=E3=80=81Newcombe=20=E4=BE=8B=E5=AD=90?= =?UTF-8?q?=E3=80=81=E5=B0=8F=E6=A0=B7=E6=9C=AC=E7=B2=BE=E7=A1=AE=E5=88=86?= =?UTF-8?q?=E5=B8=83=E5=AF=B9=E7=85=A7=E6=9A=B4=E5=8A=9B=E6=9E=9A=E4=B8=BE?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/stats.py | 467 +++++++++++++++++++++++++++ backend/app/eval/stats.py | 16 +- backend/tests/test_analysis_stats.py | 184 +++++++++++ 3 files changed, 654 insertions(+), 13 deletions(-) create mode 100644 backend/app/analysis/stats.py create mode 100644 backend/tests/test_analysis_stats.py diff --git a/backend/app/analysis/stats.py b/backend/app/analysis/stats.py new file mode 100644 index 00000000..79780b22 --- /dev/null +++ b/backend/app/analysis/stats.py @@ -0,0 +1,467 @@ +"""分析用的统计函数（docs/design/analysis-round/01-算子与判定.md §3–§5，总方案 D13）：纯 Python，不加依赖、不用 numpy。 + +为什么不装 scipy：几十 MB 的依赖只为几个函数，本机装包慢；评测本来就有一套纯 Python 的统计（Wilson 区间、精确 McNemar）， +两边要是同一套。所以 Wilson 从 eval/stats 搬到这里，评测改为引用（行为不变）。每个函数都用公认的表值钉住 +（tests/test_analysis_stats.py：正态 0.975 分位 1.959964、t(10) 双侧 0.05 临界值 2.228139、Fisher 品茶表双侧 0.4857……）。 + +- 正态：分布函数用 math.erfc；分位数用 Acklam 的有理逼近再按分布函数做两步 Halley 修正（到浮点精度）。 +- t 分布：分布函数用正则化不完全 β 函数（连分式，Lentz 法）；自由度很大（> 1e7）时和正态差在 1e-7 以内，直接按正态算； + 分位数按分布函数二分求逆。 +- 比率：Wilson 区间（缺省按评测的写法四舍五入到 4 位，places=None 不取整）、两比率之差的 Newcombe 区间（混合 score + 法，即 Newcombe 1998 的方法 10）、合并比例的 z 检验、Fisher 精确检验（超几何概率用 math.lgamma 累加，双侧按「不比 + 观测到的那张表更可能」的表求和，比较时留 1e-7 的相对余量，同 R 和 scipy）。 +- 均值：Welch t 检验，自由度按 Welch–Satterthwaite。 +- 多重比较：Holm 逐步校正。 +- 趋势：Mann–Kendall（n ≤ 10 且没有并列时用精确分布：S 的分布由逆序数的分布递推算出，不查表；否则正态近似，方差按 + 并列修正，带连续性校正）、Theil–Sen 斜率（两两斜率的中位数）。 +- 中位数、中位绝对偏差、名次。 + +能精确算的照精确算：中位数、MAD、Theil–Sen 对 Fraction 输入给 Fraction，Mann–Kendall 的精确 p 是 Fraction。浮点只出现在 +要开方、取对数、查分布的地方，结论里的数再按 analysis/numbers 统一成规范的十进制字符串。 +""" +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Sequence +from fractions import Fraction +from typing import Any, NamedTuple + +#: 95% 区间的 z（正态 0.975 分位） +Z95 = 1.959963984540054 +_SQRT2 = math.sqrt(2.0) +#: t 分布自由度大到这个数以上时按正态算（相对误差约 1/自由度） +_T_AS_NORMAL_DF = 1e7 + +# -------------------------------------------------------------------------- +# 正态分布 +# -------------------------------------------------------------------------- + + +def normal_cdf(x: float) -> float: + """标准正态分布函数 Φ(x)。""" + return 0.5 * math.erfc(-x / _SQRT2) + + +def normal_two_sided(z: float) -> float: + """标准正态的双侧尾概率 P(|Z| ≥ |z|)。""" + return math.erfc(abs(z) / _SQRT2) + + +_A = (-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, 1.383577518672690e+02, + -3.066479806614716e+01, 2.506628277459239e+00) +_B = (-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, 6.680131188771972e+01, + -1.328068155288572e+01) +_C = (-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, -2.549732539343734e+00, + 4.374664141464968e+00, 2.938163982698783e+00) +_D = (7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, 3.754408661907416e+00) + + +def normal_ppf(p: float) -> float: + """标准正态的 p 分位数 Φ⁻¹(p)。p ≤ 0 为负无穷，p ≥ 1 为正无穷。""" + if p <= 0.0: + return -math.inf + if p >= 1.0: + return math.inf + low = 0.02425 + if p < low: + q = math.sqrt(-2 * math.log(p)) + x = (((((_C[0] * q + _C[1]) * q + _C[2]) * q + _C[3]) * q + _C[4]) * q + _C[5]) / \ + ((((_D[0] * q + _D[1]) * q + _D[2]) * q + _D[3]) * q + 1) + elif p <= 1 - low: + q = p - 0.5 + r = q * q + x = (((((_A[0] * r + _A[1]) * r + _A[2]) * r + _A[3]) * r + _A[4]) * r + _A[5]) * q / \ + (((((_B[0] * r + _B[1]) * r + _B[2]) * r + _B[3]) * r + _B[4]) * r + 1) + else: + q = math.sqrt(-2 * math.log1p(-p)) + x = -(((((_C[0] * q + _C[1]) * q + _C[2]) * q + _C[3]) * q + _C[4]) * q + _C[5]) / \ + ((((_D[0] * q + _D[1]) * q + _D[2]) * q + _D[3]) * q + 1) + for _ in range(2): + # Halley 修正：误差按分布函数现算，两步就到浮点精度 + e = (normal_cdf(x) - p) if p < 0.5 else -(0.5 * math.erfc(x / _SQRT2) - (1 - p)) + u = e * math.sqrt(2 * math.pi) * math.exp(x * x / 2) + x = x - u / (1 + x * u / 2) + return x + + +# -------------------------------------------------------------------------- +# 正则化不完全 β 函数与 t 分布 +# -------------------------------------------------------------------------- + +_CF_MAX_ITER = 20_000 +_CF_EPS = 1e-15 +_CF_TINY = 1e-300 + + +def _beta_cf(a: float, b: float, x: float) -> float: + """不完全 β 函数的连分式（Lentz 法，Numerical Recipes 的 betacf）。""" + qab, qap, qam = a + b, a + 1.0, a - 1.0 + c = 1.0 + d = 1.0 - qab * x / qap + d = 1.0 / (d if abs(d) >= _CF_TINY else _CF_TINY) + h = d + for m in range(1, _CF_MAX_ITER + 1): + m2 = 2 * m + aa = m * (b - m) * x / ((qam + m2) * (a + m2)) + d = 1.0 + aa * d + d = 1.0 / (d if abs(d) >= _CF_TINY else _CF_TINY) + c = 1.0 + aa / c + c = c if abs(c) >= _CF_TINY else _CF_TINY + h *= d * c + aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2)) + d = 1.0 + aa * d + d = 1.0 / (d if abs(d) >= _CF_TINY else _CF_TINY) + c = 1.0 + aa / c + c = c if abs(c) >= _CF_TINY else _CF_TINY + step = d * c + h *= step + if abs(step - 1.0) < _CF_EPS: + break + return h + + +def betainc(a: float, b: float, x: float) -> float: + """正则化不完全 β 函数 I_x(a, b)，a、b > 0，0 ≤ x ≤ 1。""" + if x <= 0.0: + return 0.0 + if x >= 1.0: + return 1.0 + log_front = math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b) + a * math.log(x) + b * math.log1p(-x) + if x < (a + 1.0) / (a + b + 2.0): + return math.exp(log_front) * _beta_cf(a, b, x) / a + return 1.0 - math.exp(log_front) * _beta_cf(b, a, 1.0 - x) / b + + +def t_two_sided(t: float, df: float) -> float: + """自由度 df 的 t 分布的双侧尾概率 P(|T| ≥ |t|)。""" + if math.isinf(t): + return 0.0 + if df > _T_AS_NORMAL_DF or math.isinf(df): + return normal_two_sided(t) + return min(1.0, betainc(df / 2.0, 0.5, df / (df + t * t))) + + +def t_cdf(t: float, df: float) -> float: + """自由度 df 的 t 分布的分布函数。""" + tail = t_two_sided(t, df) / 2.0 + return 1.0 - tail if t >= 0 else tail + + +def t_ppf(p: float, df: float) -> float: + """自由度 df 的 t 分布的 p 分位数：按分布函数二分求逆。""" + if p <= 0.0: + return -math.inf + if p >= 1.0: + return math.inf + if p == 0.5: + return 0.0 + if df > _T_AS_NORMAL_DF or math.isinf(df): + return normal_ppf(p) + target = min(p, 1.0 - p) + lo, hi = -1.0, 0.0 + while t_cdf(lo, df) > target: + lo, hi = lo * 2.0, lo + if lo < -1e12: + break + for _ in range(200): + mid = (lo + hi) / 2.0 + if t_cdf(mid, df) > target: + hi = mid + else: + lo = mid + if hi - lo <= 1e-15 * max(1.0, abs(mid)): + break + x = (lo + hi) / 2.0 + return x if p < 0.5 else -x + + +# -------------------------------------------------------------------------- +# 比率 +# -------------------------------------------------------------------------- + + +def wilson(passed: int, n: int, z: float = Z95, *, places: int | None = 4) -> tuple[float, float] | None: + """Wilson 区间（缺省 95%）。n 为 0 时没有区间。places 是四舍五入的位数（缺省 4 位，评测一直这样显示）；None 不取整。""" + if n <= 0: + return None + p = passed / n + denom = 1 + z * z / n + centre = (p + z * z / (2 * n)) / denom + half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom + low, high = max(0.0, centre - half), min(1.0, centre + half) + if places is None: + return low, high + return round(low, places), round(high, places) + + +def newcombe(x1: int, n1: int, x2: int, n2: int, z: float = Z95) -> tuple[float, float] | None: + """两比率之差 p1 − p2 的 Newcombe 区间（混合 score 法：两边各自的 Wilson 区间合成）。有一边样本为 0 时没有区间。""" + w1, w2 = wilson(x1, n1, z, places=None), wilson(x2, n2, z, places=None) + if w1 is None or w2 is None: + return None + p1, p2 = x1 / n1, x2 / n2 + (l1, u1), (l2, u2) = w1, w2 + diff = p1 - p2 + delta = math.sqrt((p1 - l1) ** 2 + (u2 - p2) ** 2) + epsilon = math.sqrt((u1 - p1) ** 2 + (p2 - l2) ** 2) + return diff - delta, diff + epsilon + + +def pooled_z(x1: int, n1: int, x2: int, n2: int) -> tuple[float, float] | None: + """两比率相等的合并比例 z 检验：(z, 双侧 p)。有一边样本为 0 时为 None；合并比例是 0 或 1（两边全是或全不是）时 z 为 0、 + p 为 1。""" + if n1 <= 0 or n2 <= 0: + return None + pooled = (x1 + x2) / (n1 + n2) + se = math.sqrt(pooled * (1 - pooled) * (1 / n1 + 1 / n2)) + if se == 0: + return 0.0, 1.0 + z = (x1 / n1 - x2 / n2) / se + return z, normal_two_sided(z) + + +def min_expected(x1: int, n1: int, x2: int, n2: int) -> float: + """2×2 表（成功、失败 × 两组）各格的期望频数里最小的那个。""" + total = n1 + n2 + if total <= 0: + return 0.0 + successes, failures = x1 + x2, total - x1 - x2 + return min(r * c / total for r in (n1, n2) for c in (successes, failures)) + + +def _log_comb(n: int, k: int) -> float: + return math.lgamma(n + 1) - math.lgamma(k + 1) - math.lgamma(n - k + 1) + + +def fisher_exact(a: int, b: int, c: int, d: int) -> float: + """2×2 表 [[a, b], [c, d]] 的 Fisher 精确检验，双侧 p：边际固定时，不比观测到的表更可能出现的那些表的概率之和。""" + r1, r2, c1 = a + b, c + d, a + c + n = r1 + r2 + if n == 0: + return 1.0 + lo, hi = max(0, c1 - r2), min(r1, c1) + base = _log_comb(n, c1) + + def log_p(x: int) -> float: + return _log_comb(r1, x) + _log_comb(r2, c1 - x) - base + + observed = log_p(a) + 1e-7 # 相对余量 1 + 1e-7（对数里约等于加 1e-7），同 R 和 scipy + total = sum(math.exp(lp) for lp in (log_p(x) for x in range(lo, hi + 1)) if lp <= observed) + return min(1.0, total) + + +# -------------------------------------------------------------------------- +# 均值 +# -------------------------------------------------------------------------- + + +class Welch(NamedTuple): + t: float + df: float + p: float + + +def welch(m1: float, v1: float, n1: int, m2: float, v2: float, n2: int) -> Welch | None: + """两组均值相等的 Welch t 检验。v 是样本方差（n − 1 作分母）。有一组样本不到 2 个时为 None（方差算不了）。两组方差都是 0： + 均值相同 p 为 1，不同 p 为 0。""" + if n1 < 2 or n2 < 2: + return None + s1, s2 = v1 / n1, v2 / n2 + se2 = s1 + s2 + if se2 <= 0: + return Welch(0.0, math.inf, 1.0) if m1 == m2 else Welch(math.copysign(math.inf, m1 - m2), math.inf, 0.0) + t = (m1 - m2) / math.sqrt(se2) + denom = (s1 * s1 / (n1 - 1) if s1 else 0.0) + (s2 * s2 / (n2 - 1) if s2 else 0.0) + df = se2 * se2 / denom if denom > 0 else math.inf + return Welch(t, df, t_two_sided(t, df)) + + +def welch_interval(m1: float, v1: float, n1: int, m2: float, v2: float, n2: int, + level: float = 0.95) -> tuple[float, float] | None: + """m1 − m2 的 Welch 区间（缺省 95%）。算不了检验时也没有区间。""" + test = welch(m1, v1, n1, m2, v2, n2) + if test is None: + return None + se = math.sqrt(v1 / n1 + v2 / n2) + if se == 0: + return m1 - m2, m1 - m2 + q = t_ppf(0.5 + level / 2.0, test.df) + return m1 - m2 - q * se, m1 - m2 + q * se + + +# -------------------------------------------------------------------------- +# 多重比较 +# -------------------------------------------------------------------------- + + +def holm(pvalues: Sequence[float | None]) -> list[float | None]: + """Holm 逐步校正：按原始 p 从小到大，第 i 个（从 1 数）乘以 m − i + 1，再取到它为止的最大值，不超过 1。None（这一对算不了） + 不计入次数、原样留着。""" + present = sorted((p, i) for i, p in enumerate(pvalues) if p is not None) + m = len(present) + out: list[float | None] = [None] * len(pvalues) + running = 0.0 + for rank, (p, i) in enumerate(present): + running = max(running, min(1.0, (m - rank) * p)) + out[i] = running + return out + + +# -------------------------------------------------------------------------- +# 趋势 +# -------------------------------------------------------------------------- + + +def _sign(x: Any) -> int: + return (x > 0) - (x < 0) + + +class MannKendall(NamedTuple): + s: int + var: float # S 的方差（并列修正后） + z: float | None # 正态近似的 z（精确分布时为 None） + p: float | Fraction # 双侧 p；精确分布时是 Fraction + exact: bool + + +#: 精确分布最多到几期：再多 n! 太大，正态近似已经足够 +EXACT_MAX_N = 10 + + +def _inversion_counts(n: int) -> list[int]: + """1..n 的排列里，逆序数为 k 的排列有几个（k = 0 … n(n−1)/2）。递推：第 m 个数插进 m 个位置，逆序数各加 0 … m−1。""" + counts = [1] + for m in range(2, n + 1): + nxt = [0] * (len(counts) + m - 1) + for k, c in enumerate(counts): + for j in range(m): + nxt[k + j] += c + counts = nxt + return counts + + +def _scaled(values: Sequence[Any]) -> tuple[list[int], int] | None: + """一组精确的数（整数、Fraction）乘上公分母换成整数：(整数, 公分母)。有浮点的为 None。两两相减、比大小在整数上做， + 比 Fraction 快得多，结果一样。""" + if not all(isinstance(v, (int, Fraction)) and not isinstance(v, bool) for v in values): + return None + scale = math.lcm(*(Fraction(v).denominator for v in values)) if values else 1 + return [int(Fraction(v) * scale) for v in values], scale + + +def mann_kendall(values: Sequence[Any]) -> MannKendall: + """Mann–Kendall 趋势检验：S = Σ_{i 1] + var = (n * (n - 1) * (2 * n + 5) - sum(t * (t - 1) * (2 * t + 5) for t in ties)) / 18.0 + if n < 2: + return MannKendall(0, 0.0, None, Fraction(1), True) + if not ties and n <= EXACT_MAX_N: + counts = _inversion_counts(n) + pairs = n * (n - 1) // 2 + hit = sum(c for k, c in enumerate(counts) if abs(pairs - 2 * k) >= abs(s)) + return MannKendall(s, var, None, Fraction(hit, math.factorial(n)), True) + if var <= 0: + return MannKendall(s, var, 0.0, 1.0, False) + z = (s - _sign(s)) / math.sqrt(var) + return MannKendall(s, var, z, normal_two_sided(z), False) + + +def theil_sen(xs: Sequence[Any], ys: Sequence[Any]) -> Any: + """Theil–Sen 斜率：所有两点（x 不同）连线斜率的中位数。不到两个点为 None。xs、ys 都是精确的数时结果是 Fraction：斜率 + 在公分母上写成整数之比，按 int / int（正确舍入、单调）的浮点排，只把浮点相等的那几个按精确值比，n = 261 的三万多个 + 斜率也不逐个造 Fraction。""" + sx, sy = _scaled(xs), _scaled(ys) + if sx is None or sy is None: + slopes = [(ys[j] - ys[i]) / (xs[j] - xs[i]) for i in range(len(xs)) for j in range(i + 1, len(xs)) + if xs[j] != xs[i]] + return median(slopes) if slopes else None + (X, dx), (Y, dy) = sx, sy + pairs = [] + for i in range(len(X)): + for j in range(i + 1, len(X)): + den = (X[j] - X[i]) * dy + if den: + num = (Y[j] - Y[i]) * dx + pairs.append((num, den) if den > 0 else (-num, -den)) + if not pairs: + return None + keys = [num / den for num, den in pairs] + order = sorted(range(len(pairs)), key=keys.__getitem__) + + def kth(k: int) -> Fraction: + pivot = keys[order[k]] + lo, hi = k, k + 1 + while lo > 0 and keys[order[lo - 1]] == pivot: + lo -= 1 + while hi < len(order) and keys[order[hi]] == pivot: + hi += 1 + block = sorted(Fraction(*pairs[t]) for t in order[lo:hi]) + return block[k - lo] + + m = len(pairs) + return kth(m // 2) if m % 2 else (kth(m // 2 - 1) + kth(m // 2)) / 2 + + +# -------------------------------------------------------------------------- +# 位置与离散 +# -------------------------------------------------------------------------- + + +#: 超过这么多个值时中位数先按浮点排（Theil–Sen 的两两斜率有几万个，逐个按 Fraction 比太慢） +_FAST_MEDIAN = 512 + + +def _kth(values: Sequence[Any], k: int) -> Any: + """从小到大第 k 个（从 0 数），精确：Fraction 换成浮点是正确舍入的、单调的——浮点小的精确值一定小，排错的只可能是浮点 + 相等的那几个，所以先按浮点排，再只把和第 k 个浮点相等的那一段按精确值排。""" + keyed = sorted((float(v), i) for i, v in enumerate(values)) + pivot = keyed[k][0] + lo, hi = k, k + 1 + while lo > 0 and keyed[lo - 1][0] == pivot: + lo -= 1 + while hi < len(keyed) and keyed[hi][0] == pivot: + hi += 1 + block = sorted(values[i] for _, i in keyed[lo:hi]) + return block[k - lo] + + +def median(values: Sequence[Any]) -> Any: + """中位数（偶数个时取中间两个的平均）。Fraction 进 Fraction 出。没有值为 None。""" + n = len(values) + if n == 0: + return None + mid = n // 2 + if n > _FAST_MEDIAN: + upper = _kth(values, mid) + return upper if n % 2 else (_kth(values, mid - 1) + upper) / 2 + data = sorted(values) + if n % 2: + return data[mid] + return (data[mid - 1] + data[mid]) / 2 + + +def mad(values: Sequence[Any], center: Any | None = None) -> Any: + """中位绝对偏差：各值离中位数（或给定的中心）的绝对距离的中位数。""" + if not values: + return None + c = median(values) if center is None else center + return median([abs(v - c) for v in values]) + + +def rank_desc(x: Any, others: Sequence[Any]) -> int: + """x 在「others + x」里从大到小的名次（并列取最好的名次：比它大的有几个再加 1）。""" + return 1 + sum(1 for o in others if o > x) + + +__all__ = [ + "EXACT_MAX_N", "MannKendall", "Welch", "Z95", "betainc", "fisher_exact", "holm", "mad", "mann_kendall", "median", + "min_expected", "newcombe", "normal_cdf", "normal_ppf", "normal_two_sided", "pooled_z", "rank_desc", "t_cdf", + "t_ppf", "t_two_sided", "theil_sen", "welch", "welch_interval", "wilson", +] diff --git a/backend/app/eval/stats.py b/backend/app/eval/stats.py index 77f9dd22..0bfe7239 100644 --- a/backend/app/eval/stats.py +++ b/backend/app/eval/stats.py @@ -22,25 +22,15 @@ from collections import Counter from collections.abc import Iterable, Mapping, Sequence from typing import Any -#: 95% 区间的 z -Z95 = 1.959963984540054 +#: Wilson 区间和 95% 区间的 z 在分析的统计里（总方案 D13：分析和评测共用一套纯 Python 的统计），这里引用、行为不变 +from app.analysis.stats import Z95, wilson + #: 问数据页等复核最多这么久（毫秒，同 answer.REVIEW_PAGE_WAIT_S） REVIEW_PAGE_WAIT_MS = 25_000 TIERS = ("formal", "degraded", "withheld", "explore") TIMING_KEYS = ("build_ms", "run_wall_ms", "run_active_ms", "review_ms", "total_ms") -def wilson(passed: int, n: int, z: float = Z95) -> tuple[float, float] | None: - """Wilson 区间（缺省 95%）。n 为 0 时没有区间。""" - if n <= 0: - return None - p = passed / n - denom = 1 + z * z / n - centre = (p + z * z / (2 * n)) / denom - half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom - return round(max(0.0, centre - half), 4), round(min(1.0, centre + half), 4) - - def mcnemar_exact(only_a: int, only_b: int) -> float: """精确 McNemar 双侧 p 值：不一致的那几题里，「只 A 对」服从 Binomial(b + c, 0.5)。没有不一致的题时为 1。""" n = only_a + only_b diff --git a/backend/tests/test_analysis_stats.py b/backend/tests/test_analysis_stats.py new file mode 100644 index 00000000..db125315 --- /dev/null +++ b/backend/tests/test_analysis_stats.py @@ -0,0 +1,184 @@ +"""分析用的统计函数（app/analysis/stats.py）：用公认的表值钉住，外加边界——并列、n 很小、p 接近 0 / 1、t 自由度很大趋近正态。 +Wilson 从 eval/stats 搬过来，评测引用的是同一个函数、结果和以前一样。""" +from __future__ import annotations + +import itertools +import math +import random +from fractions import Fraction + +import pytest + +from app.analysis import stats +from app.eval import stats as eval_stats + + +def test_normal_quantiles_and_distribution_match_the_tables(): + assert stats.normal_ppf(0.975) == pytest.approx(1.959964, abs=1e-6) + assert stats.normal_ppf(0.975) == pytest.approx(stats.Z95, rel=1e-15) + assert stats.normal_ppf(0.5) == 0.0 + assert stats.normal_ppf(0.95) == pytest.approx(1.644854, abs=1e-6) + assert stats.normal_ppf(0.995) == pytest.approx(2.575829, abs=1e-6) + assert stats.normal_ppf(1e-10) == pytest.approx(-6.361341, abs=1e-6) # 很靠近 0 的尾巴 + assert stats.normal_ppf(0) == -math.inf and stats.normal_ppf(1) == math.inf + for p in (1e-12, 0.001, 0.02425, 0.3, 0.7, 0.97575, 0.999, 1 - 1e-9): + assert stats.normal_cdf(stats.normal_ppf(p)) == pytest.approx(p, rel=1e-9) + assert stats.normal_cdf(1.959963984540054) == pytest.approx(0.975, abs=1e-15) + assert stats.normal_two_sided(1.959963984540054) == pytest.approx(0.05, abs=1e-15) + + +def test_t_quantiles_match_the_tables(): + # 双侧 0.05 的临界值（t 分布表） + table = {1: 12.706205, 2: 4.302653, 5: 2.570582, 10: 2.228139, 30: 2.042272, 100: 1.983972} + for df, value in table.items(): + assert stats.t_ppf(0.975, df) == pytest.approx(value, abs=1e-6), df + assert stats.t_two_sided(2.228139, 10) == pytest.approx(0.05, abs=1e-6) + # 自由度 2 有闭式解：F(t) = 1/2 + t / (2√(2 + t²)) + for t in (0.3, 1.5, 4.0, 25.0): + assert stats.t_cdf(t, 2) == pytest.approx(0.5 + t / (2 * math.sqrt(2 + t * t)), rel=1e-12) + assert stats.t_cdf(0.0, 7) == 0.5 and stats.t_two_sided(0.0, 7) == 1.0 + assert stats.t_cdf(-2.0, 10) == pytest.approx(1 - stats.t_cdf(2.0, 10), rel=1e-12) + + +def test_t_with_a_huge_df_approaches_the_normal(): + assert stats.t_ppf(0.975, 1e6) == pytest.approx(stats.Z95, abs=1e-5) + assert stats.t_ppf(0.975, 1e9) == pytest.approx(stats.Z95, abs=1e-9) + assert stats.t_two_sided(1.96, 5e6) == pytest.approx(stats.normal_two_sided(1.96), abs=1e-6) + # 非整数自由度（Welch）也行 + assert stats.t_ppf(0.975, 41.784) == pytest.approx(2.018, abs=1e-3) + + +def test_regularized_incomplete_beta(): + assert stats.betainc(2, 3, 0.4) == pytest.approx(0.5248, abs=1e-12) # 多项式，手算 + assert stats.betainc(0.5, 0.5, 0.5) == pytest.approx(0.5, abs=1e-12) + assert stats.betainc(1, 1, 0.3) == pytest.approx(0.3, abs=1e-12) + assert stats.betainc(3, 2, 0.0) == 0.0 and stats.betainc(3, 2, 1.0) == 1.0 + + +def test_wilson_moved_here_is_the_same_function_the_eval_uses(): + assert eval_stats.wilson is stats.wilson + assert eval_stats.Z95 == stats.Z95 + assert stats.wilson(8, 10) == (0.4902, 0.9433) + assert stats.wilson(0, 0) is None + assert stats.wilson(0, 5)[0] == 0.0 and stats.wilson(5, 5)[1] == 1.0 + low, high = stats.wilson(8, 10, places=None) + assert round(low, 4) == 0.4902 and round(high, 4) == 0.9433 and low != 0.4902 + + +def test_newcombe_matches_the_published_example(): + # Newcombe（1998）的例子：56/70 对 48/80，混合 score 法（方法 10）0.0524 到 0.3339 + low, high = stats.newcombe(56, 70, 48, 80) + assert low == pytest.approx(0.0524, abs=1e-4) and high == pytest.approx(0.3339, abs=1e-4) + assert stats.newcombe(1, 0, 2, 5) is None + + +def test_pooled_z_and_min_expected(): + z, p = stats.pooled_z(56, 70, 48, 80) + assert z == pytest.approx(2.6502, abs=1e-4) and p == pytest.approx(0.00805, abs=1e-5) + assert stats.pooled_z(0, 10, 0, 20) == (0.0, 1.0) # 两边全不是：没有差异可言 + assert stats.pooled_z(3, 0, 1, 5) is None + assert stats.min_expected(2, 40, 5, 230) == pytest.approx(7 * 40 / 270) + + +def test_fisher_exact_two_sided(): + # Fisher 的品茶实验 + assert stats.fisher_exact(3, 1, 1, 3) == pytest.approx(0.4857142857, abs=1e-9) + assert stats.fisher_exact(1, 9, 11, 3) == pytest.approx(0.002759456, abs=1e-8) + assert stats.fisher_exact(0, 10, 10, 0) == pytest.approx(2 / math.comb(20, 10), rel=1e-9) # p 接近 0 + assert stats.fisher_exact(5, 5, 5, 5) == pytest.approx(1.0) # p 接近 1 + assert stats.fisher_exact(0, 0, 0, 0) == 1.0 + + +def test_welch_t_test(): + test = stats.welch(10, 4, 20, 8, 9, 25) + # t = 2 / √(4/20 + 9/25) = 2.6726；自由度按 Welch–Satterthwaite + assert test.t == pytest.approx(2.672612, abs=1e-6) + assert test.df == pytest.approx(0.56 ** 2 / (0.2 ** 2 / 19 + 0.36 ** 2 / 24), rel=1e-12) + assert test.p == pytest.approx(stats.t_two_sided(test.t, test.df)) + assert stats.welch(1, 1, 1, 2, 1, 5) is None # 一组只有一个样本：方差算不了 + assert stats.welch(5, 0, 3, 5, 0, 4).p == 1.0 # 没有方差、均值一样 + assert stats.welch(5, 0, 3, 6, 0, 4).p == 0.0 + low, high = stats.welch_interval(10, 4, 20, 8, 9, 25) + assert low < 2 < high and (low + high) / 2 == pytest.approx(2) + + +def test_holm_step_down(): + assert stats.holm([0.01, 0.04, 0.03, 0.005]) == pytest.approx([0.03, 0.06, 0.06, 0.02]) + assert stats.holm([0.01, None, 0.5]) == [0.02, None, 0.5] + assert stats.holm([0.6, 0.9]) == [1.0, 1.0] # 不超过 1 + assert stats.holm([]) == [] + + +def _exact_two_sided(values: list[int]) -> Fraction: + """暴力枚举全部排列，算 Mann–Kendall 的精确双侧 p（对照用）。""" + def s_of(seq: tuple[int, ...]) -> int: + return sum((b > a) - (b < a) for i, a in enumerate(seq) for b in seq[i + 1:]) + + observed = abs(s_of(tuple(values))) + perms = list(itertools.permutations(values)) + return Fraction(sum(1 for p in perms if abs(s_of(p)) >= observed), len(perms)) + + +def test_mann_kendall_small_samples_use_the_exact_distribution(): + mk = stats.mann_kendall([1, 2, 3, 4]) + assert mk.exact and mk.s == 6 and mk.p == Fraction(1, 12) # 4! 里只有两头两个排列 + assert stats.mann_kendall([1, 2, 3, 4, 5]).p == Fraction(1, 60) + for values in ([3, 1, 4, 2, 6, 5], [1, 3, 2, 5, 4, 7, 6], [2, 1, 4, 3, 6, 5, 8]): + assert stats.mann_kendall(values).p == _exact_two_sided(values) + # n = 10 的单调序列：2 / 10! + assert stats.mann_kendall(list(range(10))).p == Fraction(2, math.factorial(10)) + + +def test_mann_kendall_with_ties_or_more_points_uses_the_corrected_normal_approximation(): + values = [1, 2, 2, 3, 4, 5, 5, 6, 7, 8, 9, 10] + mk = stats.mann_kendall(values) + n = len(values) + var = (n * (n - 1) * (2 * n + 5) - 2 * (2 * 1 * 9)) / 18 # 两组各两个并列 + assert not mk.exact and mk.var == pytest.approx(var) + assert mk.z == pytest.approx((mk.s - 1) / math.sqrt(var)) # 连续性校正 + assert mk.p == pytest.approx(stats.normal_two_sided(mk.z)) + tied = stats.mann_kendall([4, 4, 4, 4, 4, 4]) # 全是并列：方差为 0 + assert tied.s == 0 and tied.p == 1.0 + small_tied = stats.mann_kendall([1, 1, 2, 3]) # n ≤ 10 但有并列：也走正态近似 + assert not small_tied.exact + + +def test_theil_sen_median_mad_and_rank(): + xs = [Fraction(i) for i in range(4)] + assert stats.theil_sen(xs, [Fraction(1), Fraction(3), Fraction(2), Fraction(6)]) == Fraction(19, 12) + assert stats.theil_sen([Fraction(0)], [Fraction(1)]) is None + assert stats.median([Fraction(3), Fraction(1), Fraction(2)]) == 2 + assert stats.median([Fraction(1), Fraction(2), Fraction(3), Fraction(10)]) == Fraction(5, 2) + assert stats.median([]) is None + assert stats.mad([Fraction(v) for v in (1, 1, 2, 2, 4, 6, 9)]) == 1 + assert stats.rank_desc(5, [1, 7, 5, 9]) == 3 and stats.rank_desc(10, [1, 2]) == 1 + + +def test_theil_sen_on_many_points_matches_the_plain_definition(): + rng = random.Random(9) + for n in (2, 3, 11, 60): + xs = [Fraction(i) for i in range(n)] + ys = [Fraction(rng.randint(-500, 500), rng.choice([1, 2, 3, 7, 100])) for _ in range(n)] + slopes = sorted((ys[j] - ys[i]) / (xs[j] - xs[i]) for i in range(n) for j in range(i + 1, n)) + m = len(slopes) + expected = slopes[m // 2] if m % 2 else (slopes[m // 2 - 1] + slopes[m // 2]) / 2 + assert stats.theil_sen(xs, ys) == expected + assert stats.theil_sen([0.0, 1.0, 2.0], [1.0, 3.0, 2.0]) == 0.5 # 浮点照样算 + # 浮点分不开的斜率：精确地比 + tiny = Fraction(1, 10 ** 25) + ys = [Fraction(0), Fraction(1) + tiny, Fraction(2) - tiny, Fraction(3)] + xs = [Fraction(i) for i in range(4)] + slopes = sorted((ys[j] - ys[i]) / (xs[j] - xs[i]) for i in range(4) for j in range(i + 1, 4)) + assert stats.theil_sen(xs, ys) == (slopes[2] + slopes[3]) / 2 + + +def test_the_fast_median_of_many_values_is_still_exact(): + rng = random.Random(5) + for size in (513, 514, 2001): + # 浮点分不开的值：1/3 加上远小于浮点精度的扰动 + values = [Fraction(1, 3) + Fraction(rng.randint(-3, 3), 10 ** 30) for _ in range(size)] + values += [Fraction(rng.randint(-1000, 1000), rng.randint(1, 260)) for _ in range(size)] + ordered = sorted(values) + n = len(ordered) + expected = ordered[n // 2] if n % 2 else (ordered[n // 2 - 1] + ordered[n // 2]) / 2 + assert stats.median(values) == expected From 654cc23c8501ab3555924f84d1a18372b153d0f0 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 13:03:23 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E5=88=86=E6=9E=90=E7=BB=93=E8=AE=BA?= =?UTF-8?q?=E7=9A=84=E6=95=B0=E4=B8=8E=E5=88=A4=E5=AE=9A=E8=AF=AD=EF=BC=9A?= =?UTF-8?q?numbers.py=EF=BC=88=E5=BF=AB=E7=85=A7=E9=87=8C=E7=9A=84?= =?UTF-8?q?=E6=95=B0=E6=8D=A2=E6=88=90=E7=B2=BE=E7=A1=AE=E7=9A=84=20Fracti?= =?UTF-8?q?on=EF=BC=8C=E5=86=99=E6=88=90=E8=A7=84=E8=8C=83=E7=9A=84?= =?UTF-8?q?=E5=8D=81=E8=BF=9B=E5=88=B6=E5=AD=97=E7=AC=A6=E4=B8=B2=EF=BC=9A?= =?UTF-8?q?=E4=B8=8D=E5=B8=A6=E6=8C=87=E6=95=B0=E3=80=81=E5=8E=BB=E5=B0=BE?= =?UTF-8?q?=E9=9B=B6=E3=80=81=E6=B2=A1=E6=9C=89=20-0=EF=BC=8C=E9=99=A4?= =?UTF-8?q?=E4=B8=8D=E5=B0=BD=E7=9A=84=E5=92=8C=E6=B5=AE=E7=82=B9=E7=95=99?= =?UTF-8?q?=2012=20=E4=BD=8D=E6=9C=89=E6=95=88=E6=95=B0=E5=AD=97=EF=BC=8C?= =?UTF-8?q?=E5=90=8C=E8=BE=93=E5=85=A5=E9=80=90=E5=AD=97=E8=8A=82=E5=90=8C?= =?UTF-8?q?=E8=BE=93=E5=87=BA=EF=BC=89=EF=BC=9Bphrases.py=20=E4=B8=80?= =?UTF-8?q?=E5=A4=84=E5=AE=9A=E4=B9=89=E5=88=A4=E5=AE=9A=E8=AF=AD=E6=A8=A1?= =?UTF-8?q?=E6=9D=BF=EF=BC=88=E7=BB=86=E6=A1=88=2001=20=C2=A77.2=EF=BC=89?= =?UTF-8?q?=E3=80=81=E6=95=B0=E7=9A=84=E5=86=99=E6=B3=95=EF=BC=88=E7=99=BE?= =?UTF-8?q?=E5=88=86=E6=AF=94=E4=B8=80=E4=BD=8D=E5=B0=8F=E6=95=B0=E5=B8=A6?= =?UTF-8?q?=E6=AD=A3=E8=B4=9F=E5=8F=B7=E3=80=81=E9=87=8F=E5=8D=83=E5=88=86?= =?UTF-8?q?=E4=BD=8D=E6=9C=80=E5=A4=9A=E4=B8=A4=E4=BD=8D=E5=B0=8F=E6=95=B0?= =?UTF-8?q?=E3=80=81=E7=99=BE=E5=88=86=E7=82=B9=EF=BC=8CROUND=5FHALF=5FUP?= =?UTF-8?q?=20=E5=90=8C=20render=5Fcell=EF=BC=89=E3=80=81label=20=E6=B6=88?= =?UTF-8?q?=E6=AF=92=EF=BC=88=E5=8E=BB=E5=BC=95=E7=94=A8=E6=A0=87=E8=AE=B0?= =?UTF-8?q?=E3=80=81=E5=87=BA=E5=85=B7=E6=A0=A1=E9=AA=8C=E8=AE=A4=E7=9A=84?= =?UTF-8?q?=E6=95=B0=E5=AD=97=E5=86=99=E6=B3=95=E3=80=81=E5=8D=95=E4=BD=8D?= =?UTF-8?q?=E7=AC=A6=E5=8F=B7=EF=BC=8C=E2=89=A4=2020=20=E5=AD=97=EF=BC=8C?= =?UTF-8?q?=E7=A9=BA=E4=BA=86=E9=80=80=E5=9B=9E=E7=9B=AE=E5=BD=95=E5=8F=AB?= =?UTF-8?q?=E6=B3=95=E5=92=8C=E5=88=97=E5=90=8D=EF=BC=89=E3=80=81=E9=99=84?= =?UTF-8?q?=E5=B8=A6=E8=AF=B4=E6=98=8E=E7=9A=84=E4=B8=80=E5=8F=A5=E8=AF=9D?= =?UTF-8?q?=EF=BC=88=E5=88=A4=E5=AE=9A=E8=AF=AD=E5=90=8E=E6=9C=80=E5=A4=9A?= =?UTF-8?q?=E6=8E=A5=E4=B8=A4=E6=9D=A1=EF=BC=89=E3=80=81=E7=85=A7=E5=AE=9E?= =?UTF-8?q?=E9=99=85=E9=98=88=E5=80=BC=E5=86=99=E7=9A=84=E5=88=A4=E5=AE=9A?= =?UTF-8?q?=E6=A0=87=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/numbers.py | 143 +++++++ backend/app/analysis/phrases.py | 517 +++++++++++++++++++++++++ backend/tests/test_analysis_phrases.py | 121 ++++++ 3 files changed, 781 insertions(+) create mode 100644 backend/app/analysis/numbers.py create mode 100644 backend/app/analysis/phrases.py create mode 100644 backend/tests/test_analysis_phrases.py diff --git a/backend/app/analysis/numbers.py b/backend/app/analysis/numbers.py new file mode 100644 index 00000000..b6d59117 --- /dev/null +++ b/backend/app/analysis/numbers.py @@ -0,0 +1,143 @@ +"""分析结论里的数：精确的有理数进、规范的十进制字符串出（细案 01 §1「数用十进制」）。 + +结论要能逐字节复算（证据面板、导出、可验证报告的复算和封存的逐字段比，评测作答前后各算一次逐字比），所以： + +- **算**：快照里的数一律换成 Fraction（浮点按最短写法 repr，数字文本按字面，DECIMAL 落成的文本不经浮点），加减乘除全程 + 精确——各段之和逐位等于总变化，Σ结构效应 + Σ自身效应逐位等于比率的变化。只有要开方、取对数、查分布的统计量是浮点。 +- **写**：canon 把数写成规范的十进制字符串：不带指数、不带正号、去掉多余的 0、没有「-0」。能精确写成有限小数的（快照里的 + 数和它们的和差，0.25 这样的商）照写；除不尽的和浮点按 SIG 位有效数字四舍五入（ROUND_HALF_UP）。浮点只留 12 位有效 + 数字：不同机器的数学库在最后一两位上可能不一样，留 12 位同一份输入在哪台机器上复算都是同一串字。 +""" +from __future__ import annotations + +import math +from decimal import Decimal, InvalidOperation +from fractions import Fraction +from typing import Any + +#: 除不尽的数、浮点保留几位有效数字 +SIG = 12 +#: 能精确写成有限小数、但有效数字超过这么多位的，也按 SIG 位写（只会出现在反复除以 2 这类极端情形） +_EXACT_MAX_DIGITS = 30 + + +def to_fraction(value: Any) -> Fraction | None: + """一个数换成精确的 Fraction：整数、Decimal、Fraction 原样；浮点按最短写法（repr）；数字文本按字面。布尔、非数、无穷、 + 认不出的为 None。""" + if isinstance(value, bool) or value is None: + return None + if isinstance(value, Fraction): + return value + if isinstance(value, int): + return Fraction(value) + if isinstance(value, float): + if not math.isfinite(value): + return None + return Fraction(Decimal(repr(value))) + if isinstance(value, Decimal): + return Fraction(value) if value.is_finite() else None + if isinstance(value, str): + try: + d = Decimal(value.strip()) + except (InvalidOperation, ValueError): + return None + return Fraction(d) if d.is_finite() else None + return None + + +def _terminating(q: Fraction) -> int | None: + """q 能精确写成有限小数时要几位小数；不能为 None。""" + d = q.denominator + twos = fives = 0 + while d % 2 == 0: + d //= 2 + twos += 1 + while d % 5 == 0: + d //= 5 + fives += 1 + return max(twos, fives) if d == 1 else None + + +def _plain(digits: int, exp10: int) -> str: + """digits × 10^exp10（digits ≥ 0）写成不带指数、去尾零的十进制。""" + text = str(digits) + if exp10 >= 0: + return text + "0" * exp10 if digits else "0" + places = -exp10 + text = text.rjust(places + 1, "0") + head, tail = text[:-places], text[-places:].rstrip("0") + return f"{head}.{tail}" if tail else head + + +def _decade(a: Fraction) -> int: + """a > 0 的十进制量级：10^e ≤ a < 10^(e+1) 的 e。""" + try: + e = math.floor(math.log10(a.numerator) - math.log10(a.denominator)) + except (ValueError, OverflowError): + e = len(str(a.numerator)) - len(str(a.denominator)) + while Fraction(10) ** e > a: + e -= 1 + while Fraction(10) ** (e + 1) <= a: + e += 1 + return e + + +def round_sig(q: Fraction, sig: int = SIG) -> Fraction: + """按 sig 位有效数字四舍五入（ROUND_HALF_UP）。""" + if q == 0: + return Fraction(0) + sign = -1 if q < 0 else 1 + a = abs(q) + e = _decade(a) + scale = Fraction(10) ** (sig - 1 - e) + m = math.floor(a * scale + Fraction(1, 2)) + return sign * Fraction(m) / scale + + +def round_places(q: Fraction, places: int) -> Fraction: + """按 places 位小数四舍五入（ROUND_HALF_UP，负数按绝对值舍入，同 Decimal 的 ROUND_HALF_UP）。""" + sign = -1 if q < 0 else 1 + scale = Fraction(10) ** places + return sign * Fraction(math.floor(abs(q) * scale + Fraction(1, 2))) / scale + + +def _write(q: Fraction) -> str: + """有限小数的 q 写出来（调用方保证 q 能精确写成有限小数）。""" + places = _terminating(q) or 0 + sign = "-" if q < 0 else "" + digits = abs(q) * (10 ** places) + assert digits.denominator == 1 + return sign + _plain(int(digits), -places) + + +def canon(value: Any) -> str | None: + """规范的十进制字符串（见文件头）。None、非数、无穷为 None。""" + if isinstance(value, float) and math.isfinite(value): + q = round_sig(to_fraction(value) or Fraction(0)) + else: + q = to_fraction(value) + if q is None: + return None + places = _terminating(q) + if places is None or len(str(int(abs(q) * 10 ** places)).rstrip("0")) > _EXACT_MAX_DIGITS: + q = round_sig(q) + if q == 0: + return "0" + return _write(q) + + +def fixed(q: Fraction, places: int, *, strip: bool = True, group: bool = False) -> str: + """按 places 位小数四舍五入写出来；strip 去掉尾零，group 加千分位。没有正号；舍成 0 的写「0」（不写「-0」）。""" + r = round_places(q, places) + sign = "-" if r < 0 else "" + scaled = abs(r) * (10 ** places) + whole, frac = divmod(int(scaled), 10 ** places) + head = f"{whole:,}" if group else str(whole) + tail = str(frac).rjust(places, "0") if places > 0 else "" + if strip: + tail = tail.rstrip("0") + text = f"{head}.{tail}" if tail else head + return sign + text + + +__all__ = ["SIG", "canon", "fixed", "round_places", "round_sig", "to_fraction"] diff --git a/backend/app/analysis/phrases.py b/backend/app/analysis/phrases.py new file mode 100644 index 00000000..438416c9 --- /dev/null +++ b/backend/app/analysis/phrases.py @@ -0,0 +1,517 @@ +"""判定语、判定标准、附带说明：一种说法，只在这里定义（细案 01 §7）。 + +判定语由系统渲染（[[a:A3]] 渲染成结论里的 phrase），写作者只引用、不改写——「叙述层无算术」往前一步：分析判定和数字一样 +由系统写。前后端都只读结论里渲染好的字符串，所以模板、数的写法、附带说明的那一句都在这里，算子只填关节。 + +**数的写法**（和 engine/evidence.render_cell 同一种风格：Decimal 式的四舍五入 ROUND_HALF_UP，千分位用「,」）： + +- 百分比：一位小数、带正负号（+8.7%、-3.1%，0 写 0.0%）；占比、比重不带正号（占 61.9%）。不是 0 的值写出来成了 0.0% + 的，多给小数位（最多 4 位），不写一个假的 0。 +- 量（金额、件数）：千分位，最多两位小数、去尾零；绝对值小于 1 的保留两位有效数字（最多 6 位小数），同样不写假的 0。 +- 百分点：比率的变化（2.1% 到 2.5% 是上升 0.4 个百分点）。 + +**label 消毒**（§7.3）：判定语里的度量叫法、因子名、维度叫法是 agent 或数据给的。agent 给的 label（以及目录里的叫法、 +当叫法用的列名）去掉数字、百分号、单位符号、引用标记，不超过 20 字，消毒后为空就退回下一个来源：不然模型可以把 +「销售额（同比 +30%）」塞进 label，借系统渲染绕过裸数字检测。去数字按出具校验抽数字的同一个函数（engine/issuance 的 +extract_numbers：阿拉伯数字、中文数字、带万 / 亿 / 成 / 倍的写法），再去掉剩下的数字字符。维度值是数据，照原样。 + +**附带说明**：判定语后面用「；」接，最多两条（按 PHRASE_ORDER 的先后），其余只在证据面板。度量类型没标、以各段之和为 +总数这类方法上的交代不进判定语。 +""" +from __future__ import annotations + +import math +import re +from collections.abc import Iterable, Mapping, Sequence +from fractions import Fraction +from typing import Any + +from app.analysis.numbers import fixed, to_fraction + +#: 算子的叫法：「算不了」的判定语、报错都用它 +OP_NAMES = {"decompose": "变化拆解", "baseline": "波动基线", "trend": "趋势判断", "differ": "差异检验", + "comparable": "可比性检查"} +#: 附带说明进判定语的先后；不在这里的（measure_unlabeled、headline_basis）只在证据面板 +PHRASE_ORDER = ("comparability", "simpson", "offset", "outlier_sensitive", "small_sample", "record_high", "record_low", + "multiple_comparisons", "weekday_effect", "small_denominator", "null_values") +#: 判定语里最多接几条附带说明 +PHRASE_CAVEATS = 2 +#: 消毒后的叫法最多几个字 +LABEL_MAX = 20 +#: 维度键有两列时，一段的写法（华东·线上） +KEY_JOIN = "·" +#: 维度值为空时的写法 +EMPTY_KEY = "（空）" + +# -------------------------------------------------------------------------- +# 拼字：汉字和英文、数字之间加一个空格，挨着全角标点的不加（同 data/sql_semantics._say） +# -------------------------------------------------------------------------- + +_CJK_PUNCT = frozenset("，。；：、！？「」『』（）《》【】…—·") + + +def _wide(ch: str) -> bool: + return "⺀" <= ch <= "鿿" or "豈" <= ch <= "﫿" or "＀" <= ch <= "￯" + + +def say(*parts: str | None) -> str: + """拼一句话：汉字和英文、数字之间补一个空格，挨着全角标点、空白的不补，汉字和汉字之间不补。""" + out = "" + for part in parts: + if not part: + continue + if out and not out[-1].isspace() and not part[0].isspace(): + a, b = out[-1], part[0] + if not (a in _CJK_PUNCT or b in _CJK_PUNCT or (_wide(a) and _wide(b)) or (not _wide(a) and not _wide(b))): + out += " " + out += part + return out + + +# -------------------------------------------------------------------------- +# 数 +# -------------------------------------------------------------------------- + + +def _q(value: Any) -> Fraction: + q = value if isinstance(value, Fraction) else to_fraction(value) + if q is None: + raise ValueError(f"不是数：{value!r}") + return q + + +def _nonzero_fixed(q: Fraction, places: int, *, most: int, strip: bool, group: bool = False) -> str: + """按 places 位小数写；不是 0 的值写出来成了 0 的，逐位多给，最多 most 位。""" + for p in range(places, most + 1): + text = fixed(q, p, strip=strip if p == places else True, group=group) + if q == 0 or text.strip("-0.,"): + return text + return fixed(q, most, strip=True, group=group) + + +def pct(value: Any, *, signed: bool = True) -> str: + """比率写成百分比：一位小数；signed 时正数带「+」（0 不带）。""" + q = _q(value) * 100 + text = _nonzero_fixed(q, 1, most=4, strip=False) + if signed and q > 0 and not text.startswith("-"): + text = "+" + text + return text + "%" + + +def points(value: Any) -> str: + """比率的变化写成百分点（不带符号，调用方写上升 / 下降）：0.004 → 0.4。""" + return _nonzero_fixed(abs(_q(value)) * 100, 1, most=4, strip=True) + + +def amount(value: Any, *, signed: bool = False) -> str: + """量：千分位，最多两位小数、去尾零；绝对值小于 1 的保留两位有效数字（最多 6 位小数）。signed 时正数带「+」。""" + q = _q(value) + places = 2 + if q != 0 and abs(q) < 1: + places = max(2, -math.floor(math.log10(float(abs(q)))) + 1) + places = min(places, 6) + text = _nonzero_fixed(q, places, most=6, strip=True, group=True) + if signed and q > 0: + text = "+" + text + return text + + +def threshold_pct(value: Any) -> str: + """阈值写成百分比：去尾零、不带正号（0.005 → 0.5%，0.8 → 80%）。""" + return fixed(_q(value) * 100, 4, strip=True) + "%" + + +def share_words(value: Any) -> str: + """阈值写进判定标准：0.5 写「一半」，其余写百分比。""" + q = _q(value) + return "一半" if q == Fraction(1, 2) else threshold_pct(q) + + +def number(value: Any) -> str: + """阈值本身（倍数、显著性水平、期数）：去尾零、不带千分位。""" + return fixed(_q(value), 6, strip=True) + + +def change(delta: Any, base: Any) -> str: + """一个量的变化：下降 194（-8.7%）/ 上升 120（+5.3%）；基期为 0 不给百分比；没变写「没有变化」。""" + d = _q(delta) + if d == 0: + return "没有变化" + word = "上升" if d > 0 else "下降" + b = _q(base) if base is not None else Fraction(0) + tail = f"（{pct(d / abs(b))}）" if b != 0 else "" + return f"{word} {amount(abs(d))}{tail}" + + +def ratio_change(r_base: Any, r_target: Any, *, proportion: bool) -> str: + """一个比率的变化。proportion（各段的分子都不超过分母，比率是一个占比，如退款率）写百分点：上升 0.4 个百分点（2.1% 到 + 2.5%）；其余（客单价这类）按量写：上升 3.2（+3.9%）。""" + rb, rt = _q(r_base), _q(r_target) + d = rt - rb + if not proportion: + return change(d, rb) + if d == 0: + return f"没有变化（{pct(rt, signed=False)}）" + word = "上升" if d > 0 else "下降" + return f"{word} {points(d)} 个百分点（{pct(rb, signed=False)} 到 {pct(rt, signed=False)}）" + + +def ratio_value(value: Any, *, proportion: bool) -> str: + """比率本身：占比写百分比，其余按量写。""" + return pct(value, signed=False) if proportion else amount(value) + + +# -------------------------------------------------------------------------- +# 叫法与段 +# -------------------------------------------------------------------------- + +_MARKER = re.compile(r"\[\[[^\]\n]*\]\]|\[\[|\]\]") +_DIGITS = re.compile(r"[0-9０-９]") +_SYMBOLS = re.compile(r"[%％‰‱¥￥$＄€£₩℃°+＋\-−–±×÷=＝<>＜＞~～^*]") +_EMPTY_BRACKETS = re.compile(r"[（(【\[]\s*[)）】\]]") +_EDGE_PUNCT = "，,、：:；;。.！!？? 　" + + +def clean_label(text: Any) -> str: + """消毒（见文件头）：去引用标记、数字（出具校验认的数字写法连同后缀，再加剩下的数字字符）、百分号和单位符号、空括号， + 压掉多余空白，不超过 LABEL_MAX 字。可能消成空串。""" + if not isinstance(text, str): + return "" + from app.engine.issuance import extract_numbers + + raw = _MARKER.sub("", text) + spans = sorted((n.start, n.end) for n in extract_numbers(raw)) + kept, pos = [], 0 + for start, end in spans: + if start >= pos: + kept.append(raw[pos:start]) + pos = end + kept.append(raw[pos:]) + out = _SYMBOLS.sub("", _DIGITS.sub("", "".join(kept))) + # 去掉数字以后留下的空白、标点：括号、标点两边的空白，括号里头尾的标点，连着的几个标点 + out = re.sub(r"\s*([（）()【】\[\]，,、：:；;])\s*", r"\1", out) + out = re.sub(r"[，,、：:；;]+(?=[）)】\]])", "", out) + out = re.sub(r"(?<=[（(【\[])[，,、：:；;]+", "", out) + out = re.sub(r"([，,、：:；;])[，,、：:；;]+", r"\1", out) + while _EMPTY_BRACKETS.search(out): + out = _EMPTY_BRACKETS.sub("", out) + out = re.sub(r"\s+", " ", out).strip(_EDGE_PUNCT) + return out[:LABEL_MAX].strip(_EDGE_PUNCT) + + +def pick_label(*candidates: Any, fallback: str = "这一列") -> str: + """依次取第一个消毒后不为空的叫法；都为空用 fallback。""" + for c in candidates: + cleaned = clean_label(c) + if cleaned: + return cleaned + return fallback + + +def key_text(key: Sequence[Any] | Any) -> str: + """一段（维度键）的写法：一列就是那个值，两列用「·」连（华东·线上）；空值写「（空）」。数据里的值照原样（压成一行）。""" + parts = key if isinstance(key, (list, tuple)) else [key] + shown = [] + for p in parts: + if p is None: + shown.append(EMPTY_KEY) + elif isinstance(p, bool): + shown.append("是" if p else "否") + else: + shown.append(re.sub(r"\s+", " ", str(p)).strip() or EMPTY_KEY) + return KEY_JOIN.join(shown) + + +def names(items: Iterable[str], limit: int | None = None) -> str: + """几段的写法：华东、华北；超过 limit 段时写前 limit 段加「等 n 段」。""" + items = list(items) + if limit is not None and len(items) > limit: + return "、".join(items[:limit]) + f"等 {len(items)} 段" + return "、".join(items) + + +def dims_text(labels: Sequence[str]) -> str: + """维度的叫法：地区；两列时写「地区、渠道」。""" + return "、".join(labels) + + +# -------------------------------------------------------------------------- +# 附带说明（caveats[].text）：一种说法 +# -------------------------------------------------------------------------- + + +def caveat(code: str, text: str) -> dict[str, str]: + return {"code": code, "text": text} + + +def offset_text(delta_total: Any, gross_up: Any, gross_down: Any) -> str: + """抵消：和总变化方向相反的段合计多少（下降的段合计 -120，上升的段合计 +40，互相抵消了一部分）。""" + up, down = amount(gross_up, signed=True), amount(gross_down, signed=True) + if _q(delta_total) >= 0: + return f"上升的段合计 {up}，下降的段合计 {down}，互相抵消了一部分" + return f"下降的段合计 {down}，上升的段合计 {up}，互相抵消了一部分" + + +def simpson_text(label: str, overall_up: bool, subject: str, w_base: Any, w_target: Any) -> str: + """辛普森：各段的比率大多和整体反着变，整体的变化来自结构。""" + seg = "上升" if not overall_up else "下降" + whole = "上升" if overall_up else "下降" + move = "升" if _q(w_target) > _q(w_base) else "降" + return say(f"各段的", label, f"大多在{seg}，整体却{whole}：变化来自结构，", subject, + f"占比从 {pct(w_base, signed=False)} {move}到 {pct(w_target, signed=False)}") + + +def small_denominator_text(segments: Sequence[str], denominator: str, minimum: int) -> str: + return say(names(segments, 3), "的", denominator, f"不到 {minimum}，比率波动大，不参与多数判断") + + +def null_segments_text(segments: Sequence[str], label: str) -> str: + return say(names(segments, 3), "的", label, "为空，按 0 算") + + +def null_periods_text(count: int, label: str) -> str: + return say(f"有 {count} 期的", label, "为空，跳过了这几期") + + +def unlabeled_text(column: str, tail: str) -> str: + return say("目录没标「", column, "」的度量类型，", tail) + + +def unconfirmed_text(column: str, measure: str, tail: str) -> str: + return say("目录里「", column, f"」是{MEASURE_WORDS.get(measure, measure)}只是推断，没人确认，", tail) + + +MEASURE_WORDS = {"flow": "流量", "stock": "存量", "ratio": "比率", "identifier": "标识", "status": "状态", "attribute": "属性"} +HEADLINE_BASIS = "没给总数，以各段之和为总数" +WEEKDAY_EFFECT = "日数据有星期效应，看趋势建议按周汇总" + + +def record_text(high: bool, label: str, n: int, unit: str) -> str: + return say("本期的", label, f"是过去 {n} {unit}里{'最高' if high else '最低'}的") + + +def multiple_text(pairs: int) -> str: + return f"比了 {pairs} 对，已按次数校正" + + +def outlier_text(largest: bool, now_differs: bool) -> str: + which = "最大" if largest else "最小" + return f"去掉{which}的一笔后，差异{'变得站得住' if now_differs else '不再站得住'}" + + +def small_sample_rate_text(minimum: int) -> str: + return f"样本少（有格子的期望频数不到 {minimum}），用了精确检验" + + +def small_sample_mean_text(group: str, n: int, minimum: int) -> str: + return say("有一组的样本数不到 ", str(minimum), "（", group, f" {n}），结论不稳") + + +UNVERIFIED_COMPLETE = "没能核对本期的数据是否到齐（查不到表里这一列的日期范围）" + + +def with_caveats(main: str, caveats: Sequence[Mapping[str, str]], *, skip: Iterable[str] = ()) -> str: + """判定语接上最多 PHRASE_CAVEATS 条附带说明（按 PHRASE_ORDER 的先后，用「；」接）。skip 里的那几条（已经在正文里说过的、 + 只在证据面板说的）不接。""" + skipped = set(skip) + ranked = sorted((PHRASE_ORDER.index(c["code"]), i, c["text"]) for i, c in enumerate(caveats) + if c["code"] in PHRASE_ORDER and c["code"] not in skipped) + texts = list(dict.fromkeys(t for _, _, t in ranked))[:PHRASE_CAVEATS] + return "；".join([main, *texts]) + + +def refused(op: str, reason: str) -> str: + """算不了的判定语：{算子的叫法}没有做成：{原因}。""" + return f"{OP_NAMES[op]}没有做成：{reason}" + + +# -------------------------------------------------------------------------- +# 判定语模板（§7.2） +# -------------------------------------------------------------------------- + + +def decompose_driver(label: str, total: str, subject: str, subject_change: str, share: Any) -> str: + return say(label, total, "，主要来自", subject, f"（{subject_change}，占 {pct(share, signed=False)}）") + + +def decompose_concentrated(label: str, total: str, subjects: str, share: Any) -> str: + return say(label, total, "，集中在", subjects, f"（合计占 {pct(share, signed=False)}）") + + +def decompose_dispersed(label: str, total: str, dims: str, main_share: Any) -> str: + return say(label, total, "，分散在各", dims, "，没有哪一段占到", share_words(main_share)) + + +def decompose_flat(label: str, pct_total: Any | None, offset: tuple[str, str, str, str] | None) -> str: + """基本持平；有抵消时接「，内部此消彼长：华东 +120、华北 -118」（最大的上升段、最大的下降段）。""" + head = say(label, "基本持平") + (f"（{pct(pct_total)}）" if pct_total is not None else "") + if offset is None: + return head + up_key, up_change, down_key, down_change = offset + return head + say("，内部此消彼长：", up_key, f" {up_change}、", down_key, f" {down_change}") + + +def decompose_unclosed(dims: str, gap: Any, *, ratio: bool) -> str: + what = "各段合起来的比率和总数对不上" if ratio else "各段之和和总数对不上" + return say("按", dims, f"拆开后{what}（差 {amount(abs(_q(gap)))}），不判断变化来自哪里") + + +def ratio_mix(label: str, total: str, subject: str, w_base: Any, w_target: Any) -> str: + return say(label, total, "，主要来自结构变化：", subject, + f"占比从 {pct(w_base, signed=False)} 到 {pct(w_target, signed=False)}") + + +def ratio_rate(label: str, total: str, subject: str) -> str: + return say(label, total, "，主要来自各段自身的变化，", subject, "影响最大") + + +def ratio_both(label: str, total: str, mix_share: Any, rate_share: Any) -> str: + return say(label, total, f"，结构变化和各段自身变化都有影响（各占 {pct(mix_share, signed=False)}、" + f"{pct(rate_share, signed=False)}）") + + +def factor_driver(label: str, total: str, factor: str, contrib: Any, share: Any) -> str: + return say(label, total, "，主要来自", factor, f"的变化（贡献 {amount(contrib, signed=True)}，占 " + f"{pct(share, signed=False)}）") + + +def factor_both(label: str, total: str, parts: Sequence[tuple[str, Any]]) -> str: + """乘积拆解里没有哪个因子占到「主要」：列出各因子占的比重。""" + shown = "、".join(say(name, f"占 {pct(share, signed=False)}") for name, share in parts) + return say(label, total, f"，几个因子都有影响（{shown}）") + + +def band_text(lo: Any, hi: Any, *, basis: str) -> str: + fmt = (lambda v: pct(v)) if basis == "change" else (lambda v: amount(v)) + return f"{fmt(lo)} 至 {fmt(hi)}" + + +def current_text(x: Any, *, basis: str, lag: int, grain: str) -> str: + """本期：变化率（和比较期比）或值本身。日数据 lag 7 写「比上周同一天」，lag 52 周 / 12 月 / 4 季写「同比」。""" + if basis == "level": + return f"本期为 {amount(x)}" + if lag == 1: + lead = "本期变化" + elif grain == "day" and lag == 7: + lead = "比上周同一天变化" + elif (grain, lag) in (("week", 52), ("month", 12), ("quarter", 4)): + lead = "同比变化" + else: + lead = f"比 {lag} 期前变化" + return f"{lead} {pct(x)}" + + +def baseline_phrase(verdict: str, label: str, current: str, n: int, unit: str, band: str) -> str: + where = {"within_band": "在过去 {n} {u}的常规波动范围内", "outside_band": "超出过去 {n} {u}的常规波动范围", + "far_outside_band": "明显超出过去 {n} {u}的常规波动范围"}[verdict].format(n=n, u=unit) + return say(label, current, f"，{where}（{band}）") + + +def baseline_flat_phrase(verdict: str, label: str, current: str, n: int, unit: str, per: str, value: str) -> str: + """历史每期都一样（稳健标准差为 0）。""" + if verdict == "within_band": + return say(label, current, f"，和过去 {n} {unit}一样（每{per}都是 {value}）") + return say(label, current, f"，明显超出过去 {n} {unit}的常规波动范围（每{per}都是 {value}）") + + +def trend_phrase(verdict: str, label: str, n: int, unit: str, per: str, slope: str | None, + run: tuple[int, str] | None) -> str: + if verdict == "no_trend": + return say(f"过去 {n} {unit}", label, "没有明显的上升或下降趋势") + word = "上升" if verdict == "rising" else "下降" + text = say(f"过去 {n} {unit}", label, f"呈{word}趋势") + (f"（每{per}约 {slope}）" if slope else "") + if run is not None: + length, direction = run + text += f"，已连续 {length} {unit}{'上升' if direction == 'up' else '下降'}" + return text + + +def differ_pair(a: str, b: str, label: str, va: str, vb: str, higher: bool) -> str: + """一对的差异站得住：西南的退款率（5.0%）高于华东（2.1%）。""" + return say(a, "的", label, f"（{va}）{'高于' if higher else '低于'}", b, f"（{vb}）") + + +def differ_phrase(pairs: Sequence[str], total: int) -> str: + """差异站得住。比了几对时写「比了 4 对，2 对差异站得住：…」。""" + if total == 1: + return f"{pairs[0]}，差异站得住" + shown = "；".join(pairs[:2]) + ("等" if len(pairs) > 2 else "") + return f"比了 {total} 对，{len(pairs)} 对差异站得住：{shown}" + + +def no_evidence_phrase(a: str | None, b: str | None, label: str, sizes: tuple[str, str] | None, total: int) -> str: + if total == 1 and a is not None and b is not None and sizes is not None: + return say(a, "和", b, "的", label, f"差异可能只是样本波动（样本量 {sizes[0]}、{sizes[1]}）") + return say(f"比了 {total} 对，", label, "的差异可能都只是样本波动") + + +def comparable_phrase(verdict: str, reasons: Sequence[str], unknown: Sequence[str]) -> str: + if verdict == "incomparable": + return "两期不可比：" + "；".join(reasons) + if verdict == "conditional": + return "两期有条件可比：" + "；".join(reasons) + return "两期可比" + (f"，{'、'.join(unknown)}核对不了" if unknown else "") + + +# -------------------------------------------------------------------------- +# 判定标准（criterion）：照实际阈值写的一句 +# -------------------------------------------------------------------------- + + +def criterion_additive(s: Mapping[str, Any], *, top_k_max: int | None, summed: bool) -> str: + k = f"最多前 {top_k_max} 段" if top_k_max else "前几段" + text = say("一段占总变化的", share_words(s["main_share"]), "以上算主要来源，", k, "合计占", + share_words(s["concentration"]), "以上算集中，总变化不到基期的", threshold_pct(s["flat_pct"]), + "算基本持平") + return text + ("，以各段之和为总数" if summed else "，各段之和要和总数对得上才判断变化来自哪里") + + +def criterion_ratio(s: Mapping[str, Any]) -> str: + return say("结构变化或各段自身变化占比率变化的", share_words(s["main_share"]), "以上算主要来自它，比率变化不到基期的", + threshold_pct(s["flat_pct"]), f"算基本持平，分母不到 {number(s['min_denominator'])} 的段不参与多数判断") + + +def criterion_factor(s: Mapping[str, Any]) -> str: + return say("按对数平均拆分（LMDI），一个因子的贡献占总变化的", share_words(s["main_share"]), + "以上算主要来自它，总变化不到基期的", threshold_pct(s["flat_pct"]), "算基本持平") + + +def criterion_baseline(s: Mapping[str, Any], n: int | None, unit: str) -> str: + span = f"过去 {n} {unit}" if n else f"过去最多 {number(s['max_history'])} 期" + warn, alert = number(s["k_warn"]), number(s["k_alert"]) + return (f"本期和{span}的中位数比，差不到 {warn} 倍稳健标准差（1.4826 倍的中位绝对偏差）算常规波动，{warn} 到 " + f"{alert} 倍算超出，{alert} 倍以上算明显超出；历史至少要 {number(s['min_history'])} 期") + + +def criterion_trend(s: Mapping[str, Any], n: int | None, unit: str) -> str: + span = f"最后 {n} {unit}" if n else f"最后最多 {number(s['max_history'])} 期" + return (f"用{span}（至少 {number(s['trend_min_points'])} 期）：Mann–Kendall 检验的双侧 p 值小于 " + f"{number(s['trend_alpha'])} 时按 Theil–Sen 斜率的正负判上升或下降，否则判没有明显趋势") + + +def criterion_differ(s: Mapping[str, Any], mode: str) -> str: + alpha = number(s["differ_alpha"]) + if mode == "rate": + return (f"双侧 p 值（比了几对时按 Holm 校正）小于 {alpha} 算差异站得住；有格子的期望频数不到 " + f"{number(s['min_expected'])} 时用 Fisher 精确检验，否则用合并比例的 z 检验") + return (f"Welch t 检验的双侧 p 值（比了几对时按 Holm 校正）小于 {alpha} 算差异站得住；一组样本数不到 " + f"{number(s['min_n'])} 算样本少") + + +CRITERION_COMPARABLE = ("只差日期、窗口一样长、是整期、星期构成相同、两边查询都没有数据问题才算可比；只有提醒项算有条件" + "可比，有一项不通过算不可比；核对不了的项不判不可比") + + +__all__ = [ + "CRITERION_COMPARABLE", "EMPTY_KEY", "HEADLINE_BASIS", "KEY_JOIN", "LABEL_MAX", "MEASURE_WORDS", "OP_NAMES", + "PHRASE_CAVEATS", "PHRASE_ORDER", "UNVERIFIED_COMPLETE", "WEEKDAY_EFFECT", "amount", "band_text", + "baseline_flat_phrase", "baseline_phrase", "caveat", "change", "clean_label", "comparable_phrase", + "criterion_additive", "criterion_baseline", "criterion_differ", "criterion_factor", "criterion_ratio", + "criterion_trend", "current_text", "decompose_concentrated", "decompose_dispersed", "decompose_driver", + "decompose_flat", "decompose_unclosed", "differ_pair", "differ_phrase", "dims_text", "factor_both", + "factor_driver", "key_text", "multiple_text", "names", "no_evidence_phrase", "null_periods_text", + "null_segments_text", "number", "offset_text", "outlier_text", "pct", "pick_label", "points", "ratio_both", + "ratio_change", "ratio_mix", "ratio_rate", "ratio_value", "record_text", "refused", "say", "share_words", + "simpson_text", "small_denominator_text", "threshold_pct", "small_sample_mean_text", "small_sample_rate_text", "trend_phrase", + "unconfirmed_text", "unlabeled_text", "with_caveats", +] diff --git a/backend/tests/test_analysis_phrases.py b/backend/tests/test_analysis_phrases.py new file mode 100644 index 00000000..147d53d8 --- /dev/null +++ b/backend/tests/test_analysis_phrases.py @@ -0,0 +1,121 @@ +"""判定语、数的写法、label 消毒、判定标准（app/analysis/phrases.py、numbers.py）：一种说法，只在这里定义。""" +from __future__ import annotations + +from fractions import Fraction + +from app.analysis import phrases as ph +from app.analysis.contract import DEFAULTS +from app.analysis.numbers import canon, fixed, round_sig, to_fraction + + +def test_canonical_decimal_strings_are_plain_and_stable(): + assert canon(Fraction(-120)) == "-120" + assert canon("2220.00") == "2220" # 去掉多余的 0 + assert canon(0.1 + 0.2) == "0.3" # 浮点留 12 位有效数字 + assert canon(Fraction(1, 3)) == "0.333333333333" + assert canon(Fraction(7, 16)) == "0.4375" # 除得尽的照写 + assert canon(-0.0) == "0" and canon(Fraction(0)) == "0" + assert canon(1e-30) == "0.000000000000000000000000000001" # 不带指数 + assert canon(1e20) == "100000000000000000000" + assert canon("123456789012345678.25") == "123456789012345678.25" # 快照里的数和它们的和差照原样 + assert canon(None) is None and canon("abc") is None and canon(True) is None + assert to_fraction(0.1) == Fraction(1, 10) and to_fraction("45678.50") == Fraction(4567850, 100) + assert round_sig(Fraction(2, 3), 3) == Fraction(667, 1000) + + +def test_fixed_rounds_half_up_like_render_cell(): + assert fixed(Fraction(2675, 1000), 2) == "2.68" # 2.675 不走浮点的 round（会得 2.67） + assert fixed(Fraction(-5, 100), 1, strip=False) == "-0.1" + assert fixed(Fraction(-4, 100), 1, strip=False) == "0.0" # 不写 -0.0 + assert fixed(Fraction(123456789, 100), 2, group=True) == "1,234,567.89" + + +def test_number_formats_in_phrases(): + assert ph.pct(Fraction(-87, 1000)) == "-8.7%" and ph.pct(Fraction(3, 100)) == "+3.0%" + assert ph.pct(0) == "0.0%" and ph.pct(Fraction(6186, 10000), signed=False) == "61.9%" + assert ph.pct(Fraction(4, 100000)) == "+0.004%" # 不是 0 的不写成 0.0% + assert ph.amount(Fraction(1234567891, 1000)) == "1,234,567.89" + assert ph.amount(Fraction(120)) == "120" and ph.amount(Fraction(-120), signed=True) == "-120" + assert ph.amount(Fraction(40), signed=True) == "+40" + assert ph.amount(Fraction(21, 1000)) == "0.021" and ph.amount(Fraction(4567, 10 ** 7)) == "0.00046" + assert ph.change(Fraction(-194), Fraction(2220)) == "下降 194（-8.7%）" + assert ph.change(Fraction(120), Fraction(0)) == "上升 120" # 基期为 0 不给百分比 + assert ph.change(0, 5) == "没有变化" + assert ph.ratio_change(Fraction(21, 1000), Fraction(25, 1000), proportion=True) == "上升 0.4 个百分点（2.1% 到 2.5%）" + assert ph.ratio_change(Fraction(82), Fraction(852, 10), proportion=False) == "上升 3.2（+3.9%）" + assert ph.threshold_pct(0.005) == "0.5%" and ph.share_words(0.5) == "一半" and ph.share_words(0.6) == "60%" + + +def test_labels_are_scrubbed_of_numbers_units_and_markers(): + assert ph.clean_label("销售额（同比 +30%）") == "销售额（同比）" + assert ph.clean_label("近三成客户的退款率") == "近客户的退款率" + assert ph.clean_label("销售额[[v:Q1.r0.gmv]]") == "销售额" + assert ph.clean_label("¥ 销售额 3.5万") == "销售额" + assert ph.clean_label("gmv_2026") == "gmv_" + assert len(ph.clean_label("销售额" * 30)) == ph.LABEL_MAX + assert ph.clean_label("30%") == "" and ph.clean_label(None) == "" + # 消毒后为空就退回下一个来源（目录里的叫法、列名） + assert ph.pick_label("12.5%", None, "gmv") == "gmv" + assert ph.pick_label("", "实收金额", "gmv") == "实收金额" + assert ph.pick_label("2026", None, "2026") == "这一列" + + +def test_numbers_stuffed_into_a_label_never_reach_the_phrase(): + label = ph.pick_label("销售额（同比 +30%，3.5万）", "销售额", "gmv") + phrase = ph.decompose_driver(label, ph.change(-194, 2220), "华东", "-120", Fraction(6186, 10000)) + assert "30" not in phrase and "3.5" not in phrase and "万" not in phrase + assert phrase == "销售额（同比）下降 194（-8.7%），主要来自华东（-120，占 61.9%）" + + +def test_spacing_between_chinese_and_ascii(): + assert ph.say("gmv", "下降 194") == "gmv 下降 194" + assert ph.say("主要来自", "online", "（", "-120") == "主要来自 online（-120" + assert ph.say("销售额", "下降") == "销售额下降" + assert ph.key_text(["华东", "线上"]) == "华东·线上" and ph.key_text([None]) == "（空）" + assert ph.names(["华东", "华北", "华南"], 2) == "华东、华北等 3 段" + + +def test_phrase_templates(): + assert ph.decompose_concentrated("销售额", "下降 194（-8.7%）", "华东、华北", Fraction(85, 100)) == \ + "销售额下降 194（-8.7%），集中在华东、华北（合计占 85.0%）" + assert ph.decompose_dispersed("销售额", "下降 194（-8.7%）", "地区", 0.5) == \ + "销售额下降 194（-8.7%），分散在各地区，没有哪一段占到一半" + assert ph.decompose_dispersed("销售额", "下降 194（-8.7%）", "地区", 0.6).endswith("没有哪一段占到 60%") + assert ph.decompose_flat("销售额", Fraction(2, 1000), ("华东", "+120", "华北", "-118")) == \ + "销售额基本持平（+0.2%），内部此消彼长：华东 +120、华北 -118" + assert ph.decompose_unclosed("地区", Fraction(-12), ratio=False) == "按地区拆开后各段之和和总数对不上（差 12），不判断变化来自哪里" + assert ph.baseline_phrase("outside_band", "销售额", "本期变化 -8.7%", 12, "周", "-6.1% 至 +5.3%") == \ + "销售额本期变化 -8.7%，超出过去 12 周的常规波动范围（-6.1% 至 +5.3%）" + assert ph.trend_phrase("rising", "线上销售额", 10, "周", "周", "+3.0%", (4, "up")) == \ + "过去 10 周线上销售额呈上升趋势（每周约 +3.0%），已连续 4 周上升" + assert ph.trend_phrase("no_trend", "销售额", 8, "个月", "月", None, None) == "过去 8 个月销售额没有明显的上升或下降趋势" + assert ph.differ_phrase([ph.differ_pair("西南", "华东", "退款率", "5.0%", "2.1%", True)], 1) == \ + "西南的退款率（5.0%）高于华东（2.1%），差异站得住" + assert ph.no_evidence_phrase("西南", "华东", "退款率", ("40", "230"), 1) == "西南和华东的退款率差异可能只是样本波动（样本量 40、230）" + assert ph.comparable_phrase("incomparable", ["本期数据只到 2026-10-15，还差 3 天"], []) == \ + "两期不可比：本期数据只到 2026-10-15，还差 3 天" + assert ph.comparable_phrase("comparable", [], ["整期"]) == "两期可比，整期核对不了" + assert ph.refused("baseline", "只有 5 期历史，至少要 8 期") == "波动基线没有做成：只有 5 期历史，至少要 8 期" + assert ph.current_text(Fraction(-31, 1000), basis="change", lag=7, grain="day") == "比上周同一天变化 -3.1%" + assert ph.current_text(Fraction(1, 10), basis="change", lag=52, grain="week") == "同比变化 +10.0%" + + +def test_caveats_follow_the_phrase_at_most_two_in_order(): + caveats = [ph.caveat("measure_unlabeled", "目录没标"), ph.caveat("null_values", "西南为空"), + ph.caveat("offset", "互相抵消"), ph.caveat("comparability", "两期有条件可比：周末不同")] + assert ph.with_caveats("判定", caveats) == "判定；两期有条件可比：周末不同；互相抵消" + assert ph.with_caveats("判定", caveats, skip=("offset",)) == "判定；两期有条件可比：周末不同；西南为空" + assert ph.with_caveats("判定", [ph.caveat("headline_basis", "以各段之和为总数")]) == "判定" + + +def test_criteria_are_written_with_the_actual_thresholds(): + text = ph.criterion_additive(DEFAULTS, top_k_max=2, summed=True) + assert text == ("一段占总变化的一半以上算主要来源，最多前 2 段合计占 80% 以上算集中，总变化不到基期的 0.5% 算基本持平，" + "以各段之和为总数") + custom = {**DEFAULTS, "main_share": 0.6, "flat_pct": 0.01} + assert "占总变化的 60% 以上" in ph.criterion_additive(custom, top_k_max=3, summed=False) + assert "1% 算基本持平" in ph.criterion_additive(custom, top_k_max=3, summed=False) + assert "差不到 2 倍稳健标准差" in ph.criterion_baseline(DEFAULTS, 12, "周") + assert "3 倍以上算明显超出" in ph.criterion_baseline(DEFAULTS, 12, "周") + assert "小于 0.05" in ph.criterion_trend(DEFAULTS, 10, "周") + assert "Fisher" in ph.criterion_differ(DEFAULTS, "rate") and "Welch" in ph.criterion_differ(DEFAULTS, "mean") From 7e6d52004c4298ac5009c530856b31b6480beb50 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 13:03:35 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E5=88=86=E6=9E=90=E7=AE=97=E5=AD=90?= =?UTF-8?q?=E7=9A=84=E8=BE=93=E5=85=A5=E3=80=81=E7=9B=AE=E5=BD=95=E4=BA=8B?= =?UTF-8?q?=E5=AE=9E=E3=80=81=E6=9C=9F=E6=AC=A1=E4=B8=8E=E7=8E=B0=E6=9F=A5?= =?UTF-8?q?=EF=BC=9Ainputs.py=EF=BC=88=E6=8C=89=E5=90=8D=E5=AD=97=E5=8F=96?= =?UTF-8?q?=E5=BF=AB=E7=85=A7=E3=80=81=E6=8B=86=E3=80=8C=E5=90=8D=E5=AD=97?= =?UTF-8?q?.r=E8=A1=8C.=E5=88=97=E3=80=8D=E3=80=81=E5=88=97=E5=90=8D?= =?UTF-8?q?=E4=B8=8D=E5=88=86=E5=A4=A7=E5=B0=8F=E5=86=99=E3=80=81=E9=81=AE?= =?UTF-8?q?=E7=BD=A9=E5=88=97=E4=B8=8E=E9=9D=9E=E6=95=B0=E5=80=BC=E6=98=AF?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E9=94=99=E8=AF=AF=EF=BC=89=EF=BC=9Bfacts.py?= =?UTF-8?q?=EF=BC=88=E6=8C=89=20SQL=20=E8=AE=A4=E7=BB=93=E6=9E=9C=E5=88=97?= =?UTF-8?q?=E5=8F=96=E8=87=AA=E5=93=AA=E5=BC=A0=E8=A1=A8=E7=9A=84=E5=93=AA?= =?UTF-8?q?=E4=B8=80=E5=88=97=EF=BC=8C=E8=AF=BB=E7=9B=AE=E5=BD=95=E7=9A=84?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E7=B1=BB=E5=9E=8B=E3=80=81=E5=8F=AB=E6=B3=95?= =?UTF-8?q?=E3=80=81=E4=B8=9A=E5=8A=A1=E6=97=A5=E6=9C=9F=EF=BC=8C=E5=8F=AA?= =?UTF-8?q?=E6=9C=89=E7=A1=AE=E8=AE=A4=E8=BF=87=E7=9A=84=E5=BA=A6=E9=87=8F?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E6=89=8D=E6=8B=BF=E6=9D=A5=E5=88=A4=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E9=94=99=E8=AF=AF=EF=BC=9B=E6=9F=A5=E8=BF=87=E7=9A=84?= =?UTF-8?q?=E8=AE=B0=E8=BF=9B=20params=EF=BC=8C=E5=A4=8D=E7=AE=97=E4=B8=8D?= =?UTF-8?q?=E8=AF=BB=E7=9B=AE=E5=BD=95=EF=BC=89=EF=BC=9Bperiods.py?= =?UTF-8?q?=EF=BC=88=E6=97=A5=E6=9C=9F=E3=80=81YYYY-MM=E3=80=81YYYY-Www?= =?UTF-8?q?=E3=80=81YYYY-Qn=E3=80=81=E6=95=B4=E6=95=B0=E5=BA=8F=E5=8F=B7?= =?UTF-8?q?=EF=BC=8C=E7=B2=92=E5=BA=A6=E6=8C=89=E5=80=BC=E6=8E=A8=E6=96=AD?= =?UTF-8?q?=EF=BC=89=EF=BC=9Bprobe.py=EF=BC=88make=5Fprober=20=E8=B5=B0=20?= =?UTF-8?q?data/coverage=20=E7=9A=84=E5=8F=AA=E8=AF=BB=E6=8E=A2=E6=9F=A5?= =?UTF-8?q?=EF=BC=9A=E9=81=AE=E7=BD=A9=E5=88=97=E4=B8=8D=E6=8E=A2=E3=80=81?= =?UTF-8?q?=E6=80=BB=E6=97=B6=E9=99=90=E3=80=815=20=E5=88=86=E9=92=9F?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E3=80=81=E9=80=80=E5=9B=9E=E5=89=96=E6=9E=90?= =?UTF-8?q?=E4=BA=8B=E5=AE=9E=EF=BC=8C=E5=8F=A6=E5=9C=A8=E5=BA=93=E9=87=8C?= =?UTF-8?q?=E7=AE=97=E6=8C=89=E8=BF=90=E8=A1=8C=E5=BD=93=E5=A4=A9=E7=8E=B0?= =?UTF-8?q?=E7=AE=97=E7=9A=84=E7=95=8C=EF=BC=9BBook=20=E5=85=88=E7=99=BB?= =?UTF-8?q?=E8=AE=B0=E3=80=81=E6=9F=A5=E5=AE=8C=E5=86=8D=E7=AE=97=EF=BC=8C?= =?UTF-8?q?=E7=BB=93=E8=AE=BA=E5=8F=AA=E8=AE=B0=E7=9C=9F=E7=94=A8=E5=88=B0?= =?UTF-8?q?=E7=9A=84=EF=BC=89=EF=BC=9Bdata/coverage.py=20=E5=8A=A0?= =?UTF-8?q?=E5=87=A0=E4=B8=AA=E5=85=AC=E5=BC=80=E7=9A=84=E5=B0=8F=E5=87=BD?= =?UTF-8?q?=E6=95=B0=EF=BC=88parse=5Fsql=E3=80=81as=5Fday=E3=80=81literal?= =?UTF-8?q?=5Fbound=E3=80=81window=5Fof=E3=80=81column=5Frange=E3=80=81bou?= =?UTF-8?q?nd=5Fvalues=E3=80=81facts=5Frange=EF=BC=89=EF=BC=8C=5Ffallback?= =?UTF-8?q?=20=E7=9A=84=E8=AF=BB=E6=B3=95=E6=8A=BD=E6=88=90=20=5Ffacts=5Fr?= =?UTF-8?q?ange=E3=80=81=E8=A1=8C=E4=B8=BA=E4=B8=8D=E5=8F=98=EF=BC=9B?= =?UTF-8?q?=E5=A5=91=E7=BA=A6=E8=A1=A5=E5=87=A0=E9=A1=B9=E5=8F=AF=E9=80=89?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=EF=BC=88InputSnapshot=20=E7=9A=84=20dialect?= =?UTF-8?q?=20/=20checks=20/=20no=5Fdata=20/=20mask=5Fcolumns=20/=20masked?= =?UTF-8?q?=5Fat=5Fquery=EF=BC=8CAnalysisInput=20=E7=9A=84=20dialect?= =?UTF-8?q?=EF=BC=8CProbe=20=E7=9A=84=20expr=20/=20value=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/contract.py | 13 +- backend/app/analysis/facts.py | 218 +++++++++++++++++++++ backend/app/analysis/inputs.py | 220 +++++++++++++++++++++ backend/app/analysis/periods.py | 224 ++++++++++++++++++++++ backend/app/analysis/probe.py | 254 +++++++++++++++++++++++++ backend/app/data/coverage.py | 94 ++++++++- backend/tests/test_analysis_periods.py | 57 ++++++ 7 files changed, 1076 insertions(+), 4 deletions(-) create mode 100644 backend/app/analysis/facts.py create mode 100644 backend/app/analysis/inputs.py create mode 100644 backend/app/analysis/periods.py create mode 100644 backend/app/analysis/probe.py create mode 100644 backend/tests/test_analysis_periods.py diff --git a/backend/app/analysis/contract.py b/backend/app/analysis/contract.py index e70d4cf8..8e82268d 100644 --- a/backend/app/analysis/contract.py +++ b/backend/app/analysis/contract.py @@ -188,6 +188,12 @@ class InputSnapshot(TypedDict, total=False): rows: list[list[Any]] truncated: bool masked: list[str] # 不能当维度、度量的列（遮罩列、按 SQL 认出改了名字的遮罩列），比较时不分大小写 + # 以下几项照查询快照里的原样带上（第 1 波 W1-a 加，都可选）： + dialect: str # SQL 的 sqlglot 方言（sqlite / mysql / postgres / oracle）；不给时按探查器的 dialect(数据源名) + checks: list[dict[str, Any]] # 快照里的 SQL 检查结果（level 为 error 的算可比性的数据问题） + no_data: list[dict[str, Any]] # 快照里的「查询区间里这张表没有数据」 + mask_columns: list[str] # 没给 masked 时（复算取回的是快照工件本身）按这两项认遮罩列 + masked_at_query: list[str] class AnalysisInput(TypedDict, total=False): @@ -201,6 +207,7 @@ class AnalysisInput(TypedDict, total=False): row: int # 一格时：第几行、哪一列、值（十进制字符串） column: str value: str | None + dialect: str # 读这份快照的 SQL 用的方言（第 1 波 W1-a 加，可选）：复算按它读，界的写法才一样 class Probe(TypedDict, total=False): @@ -213,6 +220,10 @@ class Probe(TypedDict, total=False): max: str | None basis: Literal["live", "profile", "observed"] at: str # 现查的时刻（ISO，服务端时区） + # 按运行当天现算的界（CURRENT_DATE、date('now', …)）在库里算出的值（第 1 波 W1-a 加，可选）：有 expr 的这一条记的是界， + # 没有 table / column / min / max + expr: str # 界的 SQL 写法（按这份快照的方言） + value: str | None # 库里算出的日期（带时刻的写成 YYYY-MM-DD HH:MM:SS） class Verdict(TypedDict, total=False): @@ -284,7 +295,7 @@ class Context(TypedDict, total=False): today: date # 运行当天（服务端时区）：判本期完不完整 probe: Prober | None # None 时只用 facts catalogs: Mapping[str, Any] # 数据源名 → 数据目录（app/data/catalog 的 notes）：认度量类型、业务日期列、列的叫法 - facts: Mapping[str, Any] # 数据源名 → 剖析事实（探查失败时兜底） + facts: Mapping[str, Any] # 数据源名 → 剖析事实（探查失败时兜底）：{表名: last_facts 的那一项}，或取它的函数 # -------------------------------------------------------------------------- diff --git a/backend/app/analysis/facts.py b/backend/app/analysis/facts.py new file mode 100644 index 00000000..c13d70d7 --- /dev/null +++ b/backend/app/analysis/facts.py @@ -0,0 +1,218 @@ +"""目录事实：结果列给人看的叫法、度量类型，表的业务日期（细案 01 §2.1、§6、§7.3）。 + +算子要知道「gmv 是流量还是比率」（比率当可加是调用错误；窗口不等时流量判不可比、其余只提醒）、「region 叫地区」（判定语 +写「分散在各地区」）、「orders 的业务日期是 order_date」（可比性检查按它取时间条件）。这些在数据目录里，可目录描述的是表的 +列，快照里的是 SQL 的结果列，所以先按 SQL 认结果列取自哪张表的哪一列，认法同单元格的显示规则（engine/cell_format：最外层 +SELECT 里按别名或列名找到那一项，看它是不是聚合、聚合的参数是不是一列、这一列在 FROM / JOIN 的哪张实表上），再查那一列 +的目录项： + +- SUM(x)、不聚合的 x：x 的度量类型；COUNT(*) / COUNT(x)：计数，按流量算；AVG(x)：平均值，按比率算（不能按段相加）； + COUNT(DISTINCT x)、MIN / MAX、算式：认不出（当没标）。 +- 叫法：x 的中文名（目录里的 label）；没有就是没有，调用方用列名。 +- 只认没被驳回的项（catalog.visible_notes）。度量类型另记有没有人确认过（confirmed / verified）：只有确认过的才拿来判 + 调用错误，推断的只写进附带说明（同 SQL 检查「宁可漏报，不可误报」）。 + +**复算不读目录**：run 时查过的都记进结论的 params["catalog"]（record），复算时 Facts 从那里读，同一个键得到同一个答案； +目录事后改了，封存的结论照样复算得出来。 +""" +from __future__ import annotations + +from collections.abc import Mapping +from functools import lru_cache +from typing import Any + +from app.data.names import name_key + +#: 有确证的项状态：只有它们拿来判调用错误 +_STRONG = frozenset({"confirmed", "verified"}) + + +def _notes_of(entry: Any) -> Mapping[str, Any] | None: + """目录里一张表的 notes：CatalogEntry（有 .notes）或 notes 本身；去掉被驳回的项。""" + from app.data import catalog + + notes = getattr(entry, "notes", entry) + return catalog.visible_notes(notes) if isinstance(notes, Mapping) else None + + +def _table_notes(catalog: Mapping[str, Any] | None, table: str | None) -> Mapping[str, Any] | None: + if not catalog or not table: + return None + wanted = {name_key(table), name_key(table.rsplit(".", 1)[-1])} + for name, entry in catalog.items(): + if name_key(str(name)) in wanted or name_key(str(name).rsplit(".", 1)[-1]) in wanted: + return _notes_of(entry) + return None + + +def _item(notes: Mapping[str, Any] | None, column: str, key: str) -> Mapping[str, Any] | None: + columns = (notes or {}).get("columns") if isinstance(notes, Mapping) else None + if not isinstance(columns, Mapping): + return None + for name, items in columns.items(): + if name_key(str(name)) == name_key(column) and isinstance(items, Mapping): + item = items.get(key) + return item if isinstance(item, Mapping) else None + return None + + +@lru_cache(maxsize=128) +def _outer_select(sql: str) -> Any: + """最外层的 SELECT（engine/cell_format 的读法）；同一条 SQL 只解析一次（只读，不改这棵树）。""" + from app.engine import cell_format as cf + + try: + return cf._outer_select(sql) + except Exception: # noqa: BLE001 + return None + + +def source_of(sql: str, column: str) -> tuple[str | None, str | None, str | None, bool] | None: + """结果列 column 取自哪里：(表, 列, 聚合, 去重)。聚合是 sum / count / avg / min / max / other（别的聚合）或 None（不聚合）； + 表在 SELECT * 、子查询、认不准的时候为 None。SQL 解析不了、结果列在最外层 SELECT 里找不到为 None。""" + from sqlglot import exp + + from app.engine import cell_format as cf + + try: + select = _outer_select(sql) + if select is None: + return None + item = cf._projection(select, column) + if item is None: + return None + value = cf._unwrap(item.unalias()) + agg: str | None = None + distinct = False + if isinstance(value, exp.AggFunc): + agg = {exp.Sum: "sum", exp.Count: "count", exp.Avg: "avg", exp.Min: "min", exp.Max: "max"}.get( + type(value), "other") + arg = value.this + if isinstance(arg, exp.Distinct): + distinct = True + arg = arg.expressions[0] if len(arg.expressions) == 1 else None + value = cf._unwrap(arg) if arg is not None else None + if not isinstance(value, exp.Column) or isinstance(value.this, exp.Star): + return None, None, agg, distinct + tables = cf._tables(select) + table = tables.get(name_key(value.table)) if value.table else \ + (next(iter(tables.values())) if len(tables) == 1 else None) + return table, value.name, agg, distinct + except Exception: # noqa: BLE001 - 认不出就当没标 + return None + + +def date_source(sql: str, column: str) -> tuple[str, str] | None: + """期次结果列取自哪张表的哪个日期列：那一项选取里只引用了一列（order_date、DATE(order_date)、 + strftime('%Y-%m', order_date)）时就是它。认不出为 None。""" + from sqlglot import exp + + from app.engine import cell_format as cf + + try: + select = _outer_select(sql) + item = cf._projection(select, column) if select is not None else None + if item is None: + return None + cols = [c for c in item.unalias().find_all(exp.Column) if not isinstance(c.this, exp.Star)] + if len({(name_key(c.table), name_key(c.name)) for c in cols}) != 1: + return None + ref = cols[0] + tables = cf._tables(select) + table = tables.get(name_key(ref.table)) if ref.table else \ + (next(iter(tables.values())) if len(tables) == 1 else None) + return (table, ref.name) if table else None + except Exception: # noqa: BLE001 + return None + + +class Facts: + """目录事实的取法：run 时 catalogs 是 context["catalogs"]（数据源名 → {表名: notes 或 CatalogEntry}），复算时 recorded 是 + 结论里记下的 params["catalog"]。查过的都记下（record），同一个键只认一次。""" + + def __init__(self, catalogs: Mapping[str, Any] | None = None, recorded: Mapping[str, Any] | None = None) -> None: + self._catalogs = catalogs or {} + self._recorded = recorded + self._labels: dict[str, str | None] = {} + self._measures: dict[str, tuple[str | None, bool]] = {} + self._dates: dict[str, str | None] = {} + + def _catalog(self, source: str) -> Mapping[str, Any] | None: + found = self._catalogs.get(source) + return found if isinstance(found, Mapping) else None + + def label(self, snap: Any, column: str) -> str | None: + """结果列给人看的叫法（目录里取自的那一列的中文名）；没有为 None。""" + if column in self._labels: + return self._labels[column] + if self._recorded is not None: + value = (self._recorded.get("labels") or {}).get(column) + else: + value = None + src = source_of(snap.sql, column) + if src is not None and src[1] is not None: + item = _item(_table_notes(self._catalog(snap.source), src[0]), src[1], "label") + text = item.get("value") if item else None + value = text.strip() if isinstance(text, str) and text.strip() else None + self._labels[column] = value + return value + + def measure(self, snap: Any, column: str) -> tuple[str | None, bool]: + """结果列的度量类型和有没有人确认过：(flow / stock / ratio / identifier / status / attribute 或 None, 确认过)。""" + if column in self._measures: + return self._measures[column] + if self._recorded is not None: + kind = (self._recorded.get("measures") or {}).get(column) + out = (kind, kind is not None and column not in (self._recorded.get("unconfirmed") or [])) + else: + out = self._live_measure(snap, column) + self._measures[column] = out + return out + + def _live_measure(self, snap: Any, column: str) -> tuple[str | None, bool]: + src = source_of(snap.sql, column) + if src is None: + return None, False + table, name, agg, distinct = src + if agg == "count" and not distinct: + return "flow", True + if agg == "avg": + return "ratio", True + if agg not in (None, "sum") or name is None: + return None, False + item = _item(_table_notes(self._catalog(snap.source), table), name, "measure") + kind = item.get("value") if item else None + if not isinstance(kind, str): + return None, False + return kind, item.get("status") in _STRONG + + def business_date(self, source: str, table: str) -> str | None: + """表的业务日期列（目录的 business_date）；没写为 None。""" + key = name_key(table) + if key in self._dates: + return self._dates[key] + if self._recorded is not None: + value = (self._recorded.get("dates") or {}).get(key) + else: + notes = _table_notes(self._catalog(source), table) + item = notes.get("business_date") if isinstance(notes, Mapping) else None + raw = item.get("value") if isinstance(item, Mapping) else None + col = raw.get("column") if isinstance(raw, Mapping) else None + value = col if isinstance(col, str) and col.strip() else None + self._dates[key] = value + return value + + def record(self) -> dict[str, Any] | None: + """查过、查得到的事实（写进 params["catalog"]）；一项都没有为 None。""" + out: dict[str, Any] = {} + labels = {k: v for k, v in self._labels.items() if v is not None} + measures = {k: v[0] for k, v in self._measures.items() if v[0] is not None} + unconfirmed = sorted(k for k, v in self._measures.items() if v[0] is not None and not v[1]) + dates = {k: v for k, v in self._dates.items() if v is not None} + for key, value in (("labels", labels), ("measures", measures), ("unconfirmed", unconfirmed), ("dates", dates)): + if value: + out[key] = value + return out or None + + +__all__ = ["Facts", "date_source", "source_of"] diff --git a/backend/app/analysis/inputs.py b/backend/app/analysis/inputs.py new file mode 100644 index 00000000..25d82152 --- /dev/null +++ b/backend/app/analysis/inputs.py @@ -0,0 +1,220 @@ +"""算子的输入：按名字取快照、拆单元格的写法、按列名认列、遮罩列、度量格换成精确的数、同一键同一期多行（细案 01 §1、§8.1）。 + +- 名字：agent 写本节点查过的编号（Q3，和交回模型的「【证据 Q3】」同一个），分析卡写来源节点 id；resolve 取不到就是调用错误。 +- 单元格写成「名字.r行.列」（Q1.r0.gmv），行从 0 数；名字里可以有点（取最后一个「.r数字.」切开）。 +- 列名比较不分大小写，按快照的 columns 定位：一字不差的优先；只差大小写的有好几列时是调用错误。 +- 遮罩列（快照的 masked，不分大小写）不能当维度、度量、期次：调用错误。结论里只出现非遮罩列的格。 +- 度量格：数（含 DECIMAL 落成的数字文本）换成 Fraction；空值为 None（各算子按细案 §1 处理）；文本、布尔是调用错误。 + 查询时记了类型的列按类型认：text 列里的 "2026" 不当数。 + +调用错误（contract.CallError）的说法一种，写给 agent：说清哪里不对、怎么改。 +""" +from __future__ import annotations + +import math +import re +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from fractions import Fraction +from typing import Any + +from app.analysis.contract import AnalysisInput, CallError, InputSnapshot +from app.analysis.numbers import canon, to_fraction + +#: 单元格的写法：名字.r行.列 +_CELL = re.compile(r"^(?P.+)\.r(?P\d+)\.(?P.+)$") +#: 报错里列出的列最多几个 +_LIST_MAX = 12 + + +def _listed(columns: Sequence[str]) -> str: + shown = "、".join(columns[:_LIST_MAX]) + return shown + (f"等 {len(columns)} 列" if len(columns) > _LIST_MAX else "") + + +def hashable(value: Any) -> Any: + """维度值当键：文本、数、布尔、空值原样；别的（列表、对象）和非数、无穷写成文字（同 engine/compare._hashable）。""" + if isinstance(value, float) and not math.isfinite(value): + return repr(value) + return value if isinstance(value, (str, int, float, bool)) or value is None else repr(value) + + +def same_value(a: Any, b: Any) -> bool: + """调用里写的期次、段和快照里的值是不是同一个：相等，或者写出来一样（"2026" 和 2026、" 华东" 和 "华东"）。""" + if a == b and type(a) is type(b): + return True + if isinstance(a, bool) or isinstance(b, bool): + return a == b + fa, fb = to_fraction(a), to_fraction(b) + if fa is not None and fb is not None and not isinstance(a, str) and not isinstance(b, str): + return fa == fb + return str(a).strip() == str(b).strip() + + +@dataclass(eq=False) +class Snap: + """一份输入快照（InputSnapshot 的薄包装）。""" + + ref: str + data: Mapping[str, Any] + dialect: str | None = None + columns: list[str] = field(default_factory=list) + rows: list[list[Any]] = field(default_factory=list) + + def __post_init__(self) -> None: + self.columns = [str(c) for c in self.data.get("columns") or []] + self.rows = [list(r) if isinstance(r, (list, tuple)) else [] for r in self.data.get("rows") or []] + + @property + def artifact(self) -> str | None: + value = self.data.get("artifact") + return value if isinstance(value, str) and value else None + + @property + def source(self) -> str: + return str(self.data.get("source") or "") + + @property + def sql(self) -> str: + return str(self.data.get("sql") or "") + + @property + def truncated(self) -> bool: + return bool(self.data.get("truncated")) + + @property + def masked(self) -> set[str]: + """不能当维度、度量的列（小写）：入口交来的 masked；没交（复算时取回的是快照工件本身）就按快照记下的遮罩列 + （数据源设的 mask_columns、查询时整列遮掉的 masked_at_query）。""" + found = self.data.get("masked") + if found is None: + found = [*(self.data.get("mask_columns") or []), *(self.data.get("masked_at_query") or [])] + return {str(m).strip().lower() for m in found or [] if str(m).strip()} + + def kind(self, column: str) -> str | None: + """查询时记下的这一列的类型（column_types 是 {列: 类型} 或按列序的列表）。""" + types = self.data.get("column_types") + if isinstance(types, Mapping): + kind = types.get(column) + elif isinstance(types, list) and column in self.columns: + i = self.columns.index(column) + kind = types[i] if i < len(types) else None + else: + kind = None + return kind if isinstance(kind, str) else None + + def column(self, wanted: Any, role: str) -> str: + """按名字认一列（不分大小写），认出的是快照里的写法。不存在、只差大小写的好几列、遮罩列都是调用错误。""" + if not isinstance(wanted, str) or not wanted.strip(): + raise CallError("missing_column", f"没写{role}：写 {self.ref} 里的列名") + name = wanted.strip() + hits = [c for c in self.columns if c == name] or [c for c in self.columns if c.lower() == name.lower()] + if not hits: + raise CallError("unknown_column", f"{self.ref} 里没有「{name}」这一列，{role}要写它的列名：{self.ref} 的列有 " + f"{_listed(self.columns)}") + if len(hits) > 1: + raise CallError("ambiguous_column", f"{self.ref} 里有好几列都叫「{name}」（只差大小写）：{role}写出确切的列名") + column = hits[0] + if column.lower() in self.masked: + raise CallError("masked_column", f"「{column}」是遮罩列，不能当{role}：换一列，或者在 SQL 里不选它") + return column + + def index(self, column: str) -> int: + return self.columns.index(column) + + def cell(self, row: int, column: str) -> Any: + r = self.rows[row] + i = self.index(column) + return r[i] if i < len(r) else None + + def number(self, row: int, column: str, role: str) -> Fraction | None: + """一格换成精确的数；空值为 None。不是数是调用错误。""" + raw = self.cell(row, column) + if raw is None: + return None + if isinstance(raw, bool) or (isinstance(raw, str) and self.kind(column) == "text"): + raise self._not_numeric(row, column, raw, role) + if isinstance(raw, str): + from app.engine.expressions import numeric_text + + if numeric_text(raw.strip()) is None: + raise self._not_numeric(row, column, raw, role) + value = to_fraction(raw) + if value is None: + raise self._not_numeric(row, column, raw, role) + return value + + def _not_numeric(self, row: int, column: str, raw: Any, role: str) -> CallError: + shown = str(raw).replace("\n", " ")[:20] + return CallError("not_numeric", f"{self.ref} 的「{column}」不是数值列（第 {row} 行是「{shown}」）：{role}要选数值列") + + +class Inputs: + """按名字取输入，记下结论的 inputs（同一份快照、同一格只记一次，按用到的先后）。""" + + def __init__(self, resolve: Callable[[str], InputSnapshot | None], + dialect: Callable[[str, Mapping[str, Any]], str | None] | None = None) -> None: + self._resolve = resolve + self._dialect = dialect + self._snaps: dict[str, Snap] = {} + self.records: list[AnalysisInput] = [] + + def snap(self, name: Any, role: str) -> Snap: + """按名字取一份快照。""" + if not isinstance(name, str) or not name.strip(): + raise CallError("missing_query", f"没写{role}：写本节点查过的查询编号（如 Q3）") + key = name.strip() + if key not in self._snaps: + found = self._resolve(key) + if not isinstance(found, Mapping): + raise CallError("unknown_query", f"没有叫「{key}」的查询结果：{role}写本节点查过的查询编号（如 Q3）") + self._snaps[key] = Snap(key, found, self._dialect(key, found) if self._dialect else found.get("dialect")) + return self._snaps[key] + + def cell(self, ref: Any, role: str, label: str) -> tuple[Snap, int, str, Fraction | None]: + """「名字.r行.列」→ (快照, 行, 列, 值)。""" + found = _CELL.match(ref.strip()) if isinstance(ref, str) else None + if found is None: + raise CallError("bad_cell", f"{label}要写成单元格「编号.r行.列」（如 Q1.r0.gmv），现在是「{ref}」") + snap = self.snap(found.group("name"), label) + row = int(found.group("row")) + if row >= len(snap.rows): + raise CallError("row_out_of_range", f"{snap.ref} 没有第 {row} 行（共 {len(snap.rows)} 行，从 r0 数起）：改{label}") + column = snap.column(found.group("column"), label) + value = snap.number(row, column, label) + self.record({"ref": f"{snap.ref}.r{row}.{column}", "role": role, "row": row, "column": column, + "value": canon(value)}, snap) + return snap, row, column, value + + def use(self, snap: Snap, role: str, columns: Sequence[str], rows: Sequence[int] | None = None) -> None: + """记下一份快照的用法：用了哪几列、哪几行（全用了写 all）。""" + used = sorted(set(rows)) if rows is not None else None + shown: Any = "all" if used is None or used == list(range(len(snap.rows))) else used + self.record({"ref": snap.ref, "role": role, "columns": list(dict.fromkeys(columns)), "rows": shown}, snap) + + def record(self, entry: dict[str, Any], snap: Snap) -> None: + out: dict[str, Any] = {"ref": entry["ref"]} + if snap.artifact: + out["artifact"] = snap.artifact + if snap.dialect: + out["dialect"] = snap.dialect + out.update({k: v for k, v in entry.items() if k != "ref"}) + if out not in self.records: + self.records.append(out) # type: ignore[arg-type] + + def parent(self, name: str, artifact: str) -> None: + entry: dict[str, Any] = {"ref": name, "artifact": artifact, "role": "parent"} + if entry not in self.records: + self.records.append(entry) # type: ignore[arg-type] + + +def as_list(value: Any) -> list[Any]: + """参数里一个值或一组值：都规范成列表（None 是空列表）。""" + if value is None: + return [] + if isinstance(value, (list, tuple)): + return list(value) + return [value] + + +__all__ = ["Inputs", "Snap", "as_list", "hashable", "same_value"] diff --git a/backend/app/analysis/periods.py b/backend/app/analysis/periods.py new file mode 100644 index 00000000..2f44d710 --- /dev/null +++ b/backend/app/analysis/periods.py @@ -0,0 +1,224 @@ +"""期次：快照里期次列的值认成可以排序、能判断连不连续、知道覆盖哪几天的期（细案 01 §3「期次认得出、能排序」）。 + +认这几种写法（一列里要是同一种）： + +- 日期：DATE / DATETIME（零点）、'YYYY-MM-DD'、'YYYYMMDD'、8 位整数。粒度不写时按值推断：都是某月 1 日（或都是月末） + 且相邻两期隔整月 → month（月份都在 1、4、7、10 月且隔整季 → quarter）；相邻两期都隔 7 天的整数倍 → week（这个日期当 + 一周的第一天）；否则 day。 +- 月份：'YYYY-MM'、'YYYYMM'（6 位，月份 01–12）→ month。 +- ISO 周：'YYYY-Www'（2026-W42 是 2026-10-12 到 10-18）→ week。SQLite 的 strftime('%W') 不是 ISO 周号（年初那几天 + 算第 0 周），按它写的期次覆盖的日子会差一周：方法里要求期次写成每周第一天的日期。 +- 季度：'YYYY-Qn'、'YYYYQn' → quarter。 +- 整数序号（不是上面几种的整数）→ seq：能排序、能判连续，不知道覆盖哪几天（本期完不完整核对不了）。 + +带时刻（非零点）的、写法混着的、认不出的都算认不出（period_unreadable）。 + +每一期记 ordinal：同一粒度里相邻两期的 ordinal 差 1（日：天数，周：从第一期起第几个 7 天，月：年 × 12 + 月，季：年 × 4 + +季，序号：本身），据此排序、判缺期、按 lag 找比较期。start / end 是这一期覆盖的第一天和最后一天（含），seq 没有。 +""" +from __future__ import annotations + +import calendar +import re +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from typing import Any + +from app.data import coverage + +GRAINS = ("day", "week", "month", "quarter") +#: 粒度的叫法：数量词（过去 12 周、12 个月）、每…（每周、每月） +COUNT_WORD = {"day": "天", "week": "周", "month": "个月", "quarter": "个季度", "seq": "期"} +PER_WORD = {"day": "天", "week": "周", "month": "月", "quarter": "季度", "seq": "期"} +#: 粒度不写时各粒度的缺省 lag：日数据和上周同一天比（避开星期效应），其余和上一期比 +DEFAULT_LAG = {"day": 7, "week": 1, "month": 1, "quarter": 1, "seq": 1} + +_MONTH = re.compile(r"^(\d{4})-?(\d{2})$") +_WEEK = re.compile(r"^(\d{4})-?W(\d{1,2})$", re.I) +_QUARTER = re.compile(r"^(\d{4})-?Q([1-4])$", re.I) + + +@dataclass(frozen=True) +class Period: + raw: Any # 快照里的原值 + ordinal: int + start: date | None + end: date | None + + +def _month_end(d: date) -> date: + return d.replace(day=calendar.monthrange(d.year, d.month)[1]) + + +def _quarter_start(d: date) -> date: + return date(d.year, 3 * ((d.month - 1) // 3) + 1, 1) + + +def _quarter_end(d: date) -> date: + start = _quarter_start(d) + return _month_end(date(start.year, start.month + 2, 1)) + + +def _as_date(value: Any) -> date | None: + """日期写法的期次 → 那一天；带非零点时刻的、认不出的为 None。""" + if isinstance(value, datetime): + return value.date() if value.time() == datetime.min.time() else None + if isinstance(value, date): + return value + if isinstance(value, bool): + return None + day = coverage.as_day(value) + if day is None or day.timed: + return None + return day.day + + +def _shape(value: Any) -> str | None: + """一个期次值是哪种写法：date / month / week / quarter / seq；认不出为 None。""" + if isinstance(value, bool) or value is None: + return None + if isinstance(value, (date, datetime)): + return "date" if _as_date(value) else None + if isinstance(value, float) and value.is_integer(): + value = int(value) + if isinstance(value, int): + text = str(value) + if len(text) == 8 and _as_date(value): + return "date" + if len(text) == 6 and (m := _MONTH.match(text)) and 1 <= int(m.group(2)) <= 12: + return "month" + return "seq" + if not isinstance(value, str): + return None + text = value.strip() + if (m := _MONTH.match(text)) and 1 <= int(m.group(2)) <= 12 and 1900 <= int(m.group(1)) <= 2200: + return "month" + if _WEEK.match(text): + return "week" + if _QUARTER.match(text): + return "quarter" + if _as_date(text): + return "date" + if re.fullmatch(r"-?\d+", text): + return "seq" + return None + + +def _month_of(value: Any) -> date: + m = _MONTH.match(str(int(value) if isinstance(value, (int, float)) else value).strip()) + assert m is not None + return date(int(m.group(1)), int(m.group(2)), 1) + + +def _infer_date_grain(days: Sequence[date]) -> str: + """一列日期的粒度（见文件头）。""" + ordered = sorted(set(days)) + if len(ordered) < 2: + return "day" + firsts = all(d.day == 1 for d in ordered) + lasts = all(d == _month_end(d) for d in ordered) + if firsts or lasts: + months = [d.year * 12 + d.month - 1 for d in ordered] + gaps = [b - a for a, b in zip(months, months[1:])] + if all(g >= 1 for g in gaps): + if all(d.month in (1, 4, 7, 10) for d in ordered) and all(g % 3 == 0 for g in gaps) and firsts: + return "quarter" + return "month" + gaps = [(b - a).days for a, b in zip(ordered, ordered[1:])] + if all(g % 7 == 0 for g in gaps): + return "week" + return "day" + + +def parse(values: Sequence[Any], grain: str | None = None) -> tuple[list[Period], str] | None: + """一列期次值 → (逐个的 Period，粒度)。认不出（写法不一、带时刻、粒度和写法对不上）返回 None。""" + shapes = {_shape(v) for v in values} + if not values or None in shapes or len(shapes) != 1: + return None + shape = shapes.pop() + if shape == "seq": + if grain not in (None, "seq"): + return None + return [Period(v, int(v), None, None) for v in values], "seq" + if shape == "month": + if grain not in (None, "month"): + return None + out = [] + for v in values: + start = _month_of(v) + out.append(Period(v, start.year * 12 + start.month - 1, start, _month_end(start))) + return out, "month" + if shape == "week": + if grain not in (None, "week"): + return None + out = [] + for v in values: + m = _WEEK.match(str(v).strip()) + assert m is not None + try: + start = date.fromisocalendar(int(m.group(1)), int(m.group(2)), 1) + except ValueError: + return None + out.append(Period(v, start.toordinal() // 7, start, start + timedelta(days=6))) + return out, "week" + if shape == "quarter": + if grain not in (None, "quarter"): + return None + out = [] + for v in values: + m = _QUARTER.match(str(v).strip()) + assert m is not None + start = date(int(m.group(1)), 3 * (int(m.group(2)) - 1) + 1, 1) + out.append(Period(v, start.year * 4 + (start.month - 1) // 3, start, _quarter_end(start))) + return out, "quarter" + days = [_as_date(v) for v in values] + assert all(days) + grain = grain or _infer_date_grain(days) # type: ignore[arg-type] + if grain == "day": + return [Period(v, d.toordinal(), d, d) for v, d in zip(values, days)], "day" # type: ignore[union-attr] + if grain == "week": + first = min(days) # type: ignore[type-var] + if any((d - first).days % 7 for d in days): # type: ignore[operator] + return None + return [Period(v, (d - first).days // 7, d, d + timedelta(days=6)) # type: ignore[operator] + for v, d in zip(values, days)], "week" + if grain == "month": + if not all(d.day == 1 or d == _month_end(d) for d in days): # type: ignore[union-attr] + return None + return [Period(v, d.year * 12 + d.month - 1, d.replace(day=1), _month_end(d)) # type: ignore[union-attr] + for v, d in zip(values, days)], "month" + if grain == "quarter": + if not all(d.day == 1 and d.month in (1, 4, 7, 10) for d in days): # type: ignore[union-attr] + return None + return [Period(v, d.year * 4 + (d.month - 1) // 3, d, _quarter_end(d)) # type: ignore[union-attr] + for v, d in zip(values, days)], "quarter" + return None + + +def calendar_unit(start: date, end: date) -> str | None: + """这段日子是不是一个整的日历期：整月 month、整季 quarter、整年 year；都不是为 None。整月对整月算一样长（31 天的 10 月 + 和 30 天的 9 月环比是通行的比法），不按天数判窗口不等。""" + if start.day != 1: + return None + if start.month == 1 and end == date(start.year, 12, 31): + return "year" + if start.month in (1, 4, 7, 10) and end == _quarter_end(start): + return "quarter" + if end == _month_end(start): + return "month" + return None + + +def weekend_days(start: date, end: date) -> int: + """[start, end] 里周六、周日的天数。""" + days = (end - start).days + 1 + full, rest = divmod(days, 7) + count = 2 * full + for k in range(rest): + if (start + timedelta(days=7 * full + k)).weekday() >= 5: + count += 1 + return count + + +__all__ = ["COUNT_WORD", "DEFAULT_LAG", "GRAINS", "PER_WORD", "Period", "calendar_unit", "parse", "weekend_days"] diff --git a/backend/app/analysis/probe.py b/backend/app/analysis/probe.py new file mode 100644 index 00000000..ade122d1 --- /dev/null +++ b/backend/app/analysis/probe.py @@ -0,0 +1,254 @@ +"""可比性检查的现查（细案 01 §1「只读快照、不发查询」的唯一例外，§6）：表里这一列的实际日期范围、按运行当天现算的界。 + +算子本身不查库。要现查的只有两样，都走 data/coverage 现成的只读探查（只读守卫、遮罩列不探、每条 PROBE_TIMEOUT_S 秒、总 +时限、同一张表 5 分钟内不重查、现查不成退回剖析事实）： + +- 范围：(数据源名, 表, 列) → contract.Probe {source, table, column, min, max, basis, at}，min / max 是 ISO 日期；表里这一列 + 一个值都没有时 min、max 都为 None（basis live）。basis 是 live（现查）/ profile（按剖析）/ observed（按取值观察），at 是现查的 + 时刻（服务端时区，ISO，秒）或剖析、观察的日期。 +- 界：SQL 里按运行当天现算的界（CURRENT_DATE、DATE_SUB(…)、date('now', …)）在同一个库里算出来：(数据源名, 界的 SQL 写法) + → Probe {source, expr, value, basis: live, at}，value 是库里算出的日期（带时刻的写成「YYYY-MM-DD HH:MM:SS」）。 + +**结论里记下用到的现查结果，复算用记下的值**：Book 是一次分析的现查簿。run 时算子第一遍只登记要查什么（拿到的都是「不 +知道」），入口查完再算一遍，直到不再要新的（算子在拿到范围之后才可能要界，最多三轮）；复算时 Book 里装的是结论记下的 +probes，不现查。结论的 probes 只记最后一遍真用到、查得到的，按用到的先后——同样的输入、同样的现查结果，结论逐字节相同。 + +入口（agent 分析工具、分析卡、口径卡的可比性挂钩）用 make_prober 生成 contract.Prober 交进 context["probe"]，见它的说明。 +""" +from __future__ import annotations + +import time +from collections.abc import Awaitable, Callable, Iterable, Mapping +from datetime import datetime +from typing import Any + +from app.analysis.contract import Probe +from app.data.names import name_key + +#: 算子在拿到现查结果之后可能还要新的（先要范围、再要界）：run 最多这么多轮 +MAX_ROUNDS = 3 + + +def _range_key(source: str, table: str, column: str) -> tuple[str, str, str, str]: + return ("range", source or "", name_key(str(table)), name_key(str(column))) + + +def _bound_key(source: str, expr: str) -> tuple[str, str, str]: + return ("bound", source or "", str(expr)) + + +def probe_key(probe: Mapping[str, Any]) -> tuple[str, ...]: + """一条记下的现查结果的键（Book 按它查）。""" + if probe.get("expr") is not None: + return _bound_key(str(probe.get("source") or ""), str(probe["expr"])) + return _range_key(str(probe.get("source") or ""), str(probe.get("table") or ""), str(probe.get("column") or "")) + + +class Book: + """一次分析的现查簿。known：{键: Probe 或 None（查过、查不到）}。算子经 range / bound 取；asked 是按先后登记的键，missing + 是登记了、簿里还没有的（run 据此去查），used 是最后真用到、查得到的（写进结论的 probes）。""" + + def __init__(self, known: Mapping[tuple[str, ...], Probe | None] | Iterable[Probe] = ()) -> None: + if isinstance(known, Mapping): + self._known: dict[tuple[str, ...], Probe | None] = dict(known) + self._closed = False + else: + self._known = {probe_key(p): p for p in known} + self._closed = True # 复算：记下的就是全部，没记的当查不到 + self.asked: list[tuple[str, ...]] = [] + + def _ask(self, key: tuple[str, ...]) -> Probe | None: + if key not in self.asked: + self.asked.append(key) + return self._known.get(key) + + def range(self, source: str, table: str, column: str) -> Probe | None: + return self._ask(_range_key(source, table, column)) + + def bound(self, source: str, expr: str) -> Probe | None: + return self._ask(_bound_key(source, expr)) + + @property + def missing(self) -> list[tuple[str, ...]]: + return [] if self._closed else [k for k in self.asked if k not in self._known] + + @property + def used(self) -> list[Probe]: + return [p for p in (self._known.get(k) for k in self.asked) if p is not None] + + +def _now() -> str: + return datetime.now().astimezone().replace(tzinfo=None).isoformat(timespec="seconds") + + +async def fetch(book: Book, context: Mapping[str, Any]) -> dict[tuple[str, ...], Probe | None]: + """把簿里登记了、还没有的现查一遍：先 context["probe"]，查不到的范围退回 context["facts"]（数据源名 → {表名: 剖析事实}， + 或数据源名 → 取剖析事实的函数），都没有记 None。交回新查到的那几条。""" + from app.data import coverage + + prober = context.get("probe") + facts = context.get("facts") or {} + out: dict[tuple[str, ...], Probe | None] = {} + for key in book.missing: + found: Probe | None = None + if key[0] == "range": + _, source, table, column = key + if prober is not None: + try: + found = await prober(source, table, column) + except Exception: # noqa: BLE001 - 探查失败就当不知道 + found = None + if found is None: + found = await _from_facts(facts.get(source), source, table, column, coverage) + else: + _, source, expr = key + bounds = getattr(prober, "bounds", None) + if bounds is not None: + try: + found = await bounds(source, expr) + except Exception: # noqa: BLE001 + found = None + out[key] = found + return out + + +async def _from_facts(facts: Any, source: str, table: str, column: str, coverage: Any) -> Probe | None: + """探查不成时按剖析事实兜底：facts 是 {表名: last_facts 的那一项} 或取它的函数（coverage.Facts）。""" + found: Any = None + try: + if callable(facts): + found = await facts(table) + elif isinstance(facts, Mapping): + found = next((v for k, v in facts.items() if name_key(str(k)) == name_key(table) + or name_key(str(k).rsplit(".", 1)[-1]) == name_key(table.rsplit(".", 1)[-1])), None) + except Exception: # noqa: BLE001 + return None + data = coverage.facts_range(found, column) + return _range_probe(source, table, column, data) if data else None + + +def _range_probe(source: str, table: str, column: str, data: Mapping[str, Any]) -> Probe: + basis = data.get("basis") if data.get("basis") in ("live", "profile", "observed") else "live" + at = _now() if basis == "live" else str(data.get("at") or "") + return {"source": source, "table": table, "column": column, "min": data.get("min"), "max": data.get("max"), + "basis": basis, "at": at} # type: ignore[typeddict-item] + + +class LiveProber: + """contract.Prober：await prober(数据源名, 表, 列) → Probe | None；另有 await prober.bounds(数据源名, 界的 SQL) 在库里算界， + prober.dialect(数据源名) 是这个源的 sqlglot 方言（算子按它读 SQL、写界）。见 make_prober。""" + + def __init__(self, sources: Mapping[str, Any], *, masked: Mapping[str, Iterable[str]] | None, + facts: Mapping[str, Callable[[str], Awaitable[Mapping[str, Any] | None]]] | None, + cache: Mapping[str, Any] | None, budget: float, + clock: Callable[[], str] | None) -> None: + from app.data import coverage + + self._sources = dict(sources) + self._masked = {k: {str(c).strip().lower() for c in v} for k, v in (masked or {}).items()} + self._facts = dict(facts or {}) + self._cache: dict[str, Any] = dict(cache or {}) + self._budget = budget + self._deadline: float | None = None + self._clock = clock or _now + self._bounds: dict[tuple[str, str], Any] = {} + self._coverage = coverage + + def dialect(self, source: str) -> str | None: + from app.data.sqlcheck import dialect_of + + found = self._sources.get(source) + return dialect_of(getattr(found, "kind", None)) if found is not None else None + + def _expired(self) -> bool: + if self._deadline is None: + self._deadline = time.monotonic() + self._budget + return time.monotonic() > self._deadline + + def _hidden(self, source: Any, name: str) -> set[str]: + if name in self._masked: + return self._masked[name] + from app.data.engine import masked_columns + + return {c.lower() for c in masked_columns(getattr(source, "options", None) or {})} + + def _facts_of(self, name: str, source: Any) -> Callable[[str], Awaitable[Mapping[str, Any] | None]]: + return self._facts.get(name) or profile_facts(source) + + async def __call__(self, source: str, table: str, column: str) -> Probe | None: + found = self._sources.get(source) + if found is None or str(column).strip().lower() in self._hidden(found, source): + return None + if self._expired(): + return None + cache = self._cache.setdefault(source, self._coverage.Cache()) + data = await self._coverage.column_range(found, table, column, facts=self._facts_of(source, found), cache=cache) + if data is None: + return None + probe = _range_probe(source, table, column, data) + if probe["basis"] == "live": + probe["at"] = self._clock() + return probe + + async def bounds(self, source: str, expr: str) -> Probe | None: + found = self._sources.get(source) + if found is None or self._expired(): + return None + key = (source, expr) + if key not in self._bounds: + values = await self._coverage.bound_values(found, [expr]) + self._bounds[key] = values[0] if values else None + value = self._bounds[key] + if value is None: + return None + return {"source": source, "expr": expr, "value": str(value), "basis": "live", # type: ignore[typeddict-unknown-key] + "at": self._clock()} + + +def profile_facts(source: Any) -> Callable[[str], Awaitable[Mapping[str, Any] | None]]: + """这个数据源上一张表的剖析事实（catalog_profile_jobs.last_facts 的那一项，同 tools/datasource._coverage 的退回）：上传 + 表格的快照对不上这次读的那一版时不用。""" + async def facts(table: str) -> Mapping[str, Any] | None: + from app.data import catalog as data_catalog + from app.data.catalog_profile_jobs import last_facts + from app.db.base import SessionLocal + + source_id = getattr(source, "id", None) + if not source_id: + return None + name = data_catalog.resolve_table_name(getattr(source, "schema_cache", None), table) or table + async with SessionLocal() as session: + found = (await last_facts(session, str(source_id), [name])).get(name) + snapshot = getattr(source, "snapshot_id", None) + if found and snapshot and found.get("snapshot_id") != snapshot: + return None + return found + + return facts + + +def make_prober(sources: Mapping[str, Any], *, masked: Mapping[str, Iterable[str]] | None = None, + facts: Mapping[str, Callable[[str], Awaitable[Mapping[str, Any] | None]]] | None = None, + cache: Mapping[str, Any] | None = None, budget: float | None = None, + clock: Callable[[], str] | None = None) -> LiveProber: + """生成一次分析调用用的探查器（contract.Prober，交进 context["probe"]）。 + + - sources：数据源名 → 数据源对象（DataSource 行，或上传源绑定快照的视图：同 tools/datasource 建工具时的 source，要有 + name、kind、options、schema_cache；上传源另有 snapshot_id）。名字和快照里的 source 一致。没给的源不探。 + - masked：数据源名 → 遮罩列（不分大小写）。缺省按数据源 options 里设的遮罩列（data/engine.masked_columns）。遮罩列上 + 不探，探查器直接交回 None。 + - facts：数据源名 → 取剖析事实的函数（coverage.Facts：表名 → last_facts 的那一项）。缺省按 profile_facts 现读库（同查询 + 工具的覆盖检查）。 + - cache：数据源名 → coverage.Cache（同一张表、同一个界一次运行里只查一次）；不给就每个探查器各自一份。另外 coverage + 在进程里把现查到的范围记 5 分钟。 + - budget：这个探查器所有探查加起来最多多少秒（缺省 coverage.BUDGET_S，从第一次探查算起）；过了的一律交回 None。一次 + 分析调用建一个。 + - clock：现查时刻的写法（测试用）；缺省是服务端时区的 ISO 时间（到秒）。 + """ + from app.data import coverage + + return LiveProber(sources, masked=masked, facts=facts, cache=cache, + budget=coverage.BUDGET_S if budget is None else budget, clock=clock) + + +__all__ = ["Book", "LiveProber", "MAX_ROUNDS", "fetch", "make_prober", "probe_key", "profile_facts"] diff --git a/backend/app/data/coverage.py b/backend/app/data/coverage.py index e7db5445..e30761b9 100644 --- a/backend/app/data/coverage.py +++ b/backend/app/data/coverage.py @@ -447,9 +447,14 @@ async def _fallback(flt: _Filter, facts: Facts) -> _Range | None: found = await facts(flt.table_name) except Exception: # noqa: BLE001 return None + return _facts_range(found, flt.column_name) + + +def _facts_range(found: Any, column: str) -> _Range | None: + """剖析事实（last_facts 的一项）里这一列的范围：先剖析的 date_ranges，再取值观察的最新日期。""" if not isinstance(found, Mapping): return None - wanted = name_key(flt.column_name) + wanted = name_key(column) for item in found.get("date_ranges") or []: if isinstance(item, Mapping) and name_key(str(item.get("column") or "")) == wanted: lo, hi = _as_day(item.get("min")), _as_day(item.get("max")) @@ -806,5 +811,88 @@ def no_data_of(snapshot: Any, column: Any) -> dict[str, Any] | None: return None -__all__ = ["BUDGET_S", "Cache", "NO_DATA_SHORT", "PROBE_TIMEOUT_S", "Window", "filters", "narrowed", "no_data_of", - "result_gaps", "sql_gaps", "zero_columns"] +# ========================================================================== +# 给分析的可比性检查用的几个小函数（app/analysis/comparable.py、probe.py：窗口长度、整期）。读法、写法、探查都和上面同一套， +# 只是把中间结果交出去：分析结论要把现查的范围、在库里算出的界记下来，复算时用记下的值 +# ========================================================================== + + +def parse_sql(sql: Any, read: str | None) -> Any: + """一条查询解析成语法树（同覆盖检查：太长、带模板占位、不止一条、不是查询的为 None）。""" + return _parse(sql, read) + + +def as_day(value: Any, kind: str | None = None) -> _Day | None: + """一个值归一成日期（DATE / DATETIME、'YYYY-MM-DD'、'YYYYMMDD'、8 位整数）；认不出为 None。""" + return _as_day(value, kind) + + +def literal_bound(node: Any) -> tuple[bool, _Day | None]: + """时间条件的一个界：(是不是字面量, 字面量归一成的日期)。不是字面量的是按运行当天现算的，要在库里算。""" + return (True, _literal_day(node)) if _is_literal(node) else (False, None) + + +def window_of(conds: list[tuple[str, Any, bool]], days: Mapping[int, _Day | None]) -> Window | None: + """几个条件合成一段区间（同 _window：下界取最晚、上界取最早，「>」「<」的界不带时刻时按天挪一天）。days 是 + {id(界的节点): 日期}。不核对列和界的写法比不比得起来：分析结论里表的范围只记了日期。""" + return _window(conds, days, _Range(None, None, "none"), None) + + +def _range_out(data: _Range | None) -> dict[str, Any] | None: + if data is None: + return None + return {"min": data.low.day.isoformat() if data.low else None, + "max": data.high.day.isoformat() if data.high else None, + "basis": data.basis, "at": data.day, "empty": data.empty} + + +def facts_range(found: Any, column: str) -> dict[str, Any] | None: + """剖析事实（catalog_profile_jobs.last_facts 的一项）里这一列的范围，形状同 column_range；没有为 None。""" + return _range_out(_facts_range(found, column)) + + +async def column_range(source: Any, table: str, column: str, *, facts: Facts | None = None, + cache: Cache | None = None) -> dict[str, Any] | None: + """这一列在这张表里的实际范围：现查 MIN / MAX（只读守卫、PROBE_TIMEOUT_S 秒、同一张表 LIVE_TTL_S 秒内不重查），查不成 + 退回剖析、取值观察（facts）。{min, max, basis, at, empty}：min / max 是 ISO 日期（不知道的那一头为 None），basis 是 live / + profile / observed，at 是剖析、观察的日期（现查为 None），表里这一列一个值都没有时 empty 为真。都没有为 None。从不抛异常。""" + from sqlglot import exp + + try: + parts = [p for p in str(table).split(".") if p] + node = exp.Table(this=exp.to_identifier(parts[-1]), + **({"db": exp.to_identifier(parts[-2])} if len(parts) > 1 else {})) + flt = _Filter((), node, str(table), exp.column(str(column)), str(column)) + return _range_out(await _range(source, flt, _read(getattr(source, "kind", None)), cache or Cache(), facts)) + except Exception: # noqa: BLE001 - 探查失败就当不知道 + logger.info("可比性检查的范围探查没有成功（数据源 %s）", getattr(source, "name", ""), exc_info=True) + return None + + +async def bound_values(source: Any, texts: list[str]) -> list[Any] | None: + """按运行当天现算的几个界（SQL 表达式，按这个数据源的方言写）在库里算出来：一条 SELECT 界1, 界2 …（Oracle 加 FROM + DUAL），走只读守卫和 PROBE_TIMEOUT_S。交回各个值（日期、时刻按 ISO 写，其余照库里交回的）；查不成为 None。从不抛异常。""" + if not texts: + return [] + read = _read(getattr(source, "kind", None)) + try: + found = await _probe(source, _select_sql([f"{t} AS b{i}" for i, t in enumerate(texts)], read)) + except Exception: # noqa: BLE001 + return None + if found is None: + return None + values = list(found[0]) + [None] * len(texts) + out: list[Any] = [] + for value in values[:len(texts)]: + if isinstance(value, datetime): + out.append(value.isoformat(sep=" ", timespec="seconds")) + elif isinstance(value, date): + out.append(value.isoformat()) + else: + out.append(value) + return out + + +__all__ = ["BUDGET_S", "Cache", "NO_DATA_SHORT", "PROBE_TIMEOUT_S", "Window", "as_day", "bound_values", "column_range", + "facts_range", "filters", "literal_bound", "narrowed", "no_data_of", "parse_sql", "result_gaps", "sql_gaps", + "window_of", "zero_columns"] diff --git a/backend/tests/test_analysis_periods.py b/backend/tests/test_analysis_periods.py new file mode 100644 index 00000000..eee79dde --- /dev/null +++ b/backend/tests/test_analysis_periods.py @@ -0,0 +1,57 @@ +"""期次（app/analysis/periods.py）：日期、YYYY-MM、YYYY-Www、YYYY-Qn、整数序号认得出、能排序、知道覆盖哪几天；粒度按值推断； +写法混着、带时刻的认不出。""" +from __future__ import annotations + +from datetime import date + +from app.analysis import periods as pd + + +def test_dates_infer_their_grain(): + items, grain = pd.parse(["2026-10-01", "2026-10-02", "2026-10-04"]) + assert grain == "day" and [i.start for i in items] == [date(2026, 10, 1), date(2026, 10, 2), date(2026, 10, 4)] + items, grain = pd.parse(["2026-09-28", "2026-10-05", "2026-10-19"]) # 隔 7 天的整数倍：周（缺一周） + assert grain == "week" and [i.ordinal for i in items] == [0, 1, 3] and items[0].end == date(2026, 10, 4) + items, grain = pd.parse(["2026-08-01", "2026-09-01", "2026-10-01"]) + assert grain == "month" and items[1].end == date(2026, 9, 30) + items, grain = pd.parse(["2026-01-01", "2026-04-01", "2026-07-01"]) + assert grain == "quarter" and items[0].end == date(2026, 3, 31) + items, grain = pd.parse([20261001, 20261002]) + assert grain == "day" and items[1].start == date(2026, 10, 2) + assert pd.parse(["2026-10-01 00:00:00", "2026-10-02 00:00:00"])[1] == "day" + + +def test_labels_for_months_weeks_quarters_and_sequences(): + items, grain = pd.parse(["2026-09", "2026-10"]) + assert grain == "month" and items[1].ordinal - items[0].ordinal == 1 and items[1].end == date(2026, 10, 31) + items, grain = pd.parse([202609, 202610]) + assert grain == "month" + items, grain = pd.parse(["2026-W41", "2026-W42", "2026-W01"]) + assert grain == "week" and items[1].start == date(2026, 10, 12) and items[1].end == date(2026, 10, 18) + assert items[1].ordinal - items[0].ordinal == 1 + items, grain = pd.parse(["2025-W52", "2026-W01"]) # 跨年也连着 + assert items[1].ordinal - items[0].ordinal == 1 + items, grain = pd.parse(["2026Q3", "2026-Q4"]) + assert grain == "quarter" and items[1].start == date(2026, 10, 1) + items, grain = pd.parse([3, 1, 2]) + assert grain == "seq" and items[0].start is None and [i.ordinal for i in items] == [3, 1, 2] + + +def test_unreadable_periods(): + assert pd.parse(["2026-10-01", "2026-W41"]) is None # 写法混着 + assert pd.parse(["2026-10-01 08:30:00", "2026-10-02 08:30:00"]) is None # 带时刻 + assert pd.parse(["上周", "本周"]) is None + assert pd.parse(["2026-10", "2026-11"], "week") is None # 粒度和写法对不上 + assert pd.parse(["2026-10-01", "2026-10-03"], "week") is None # 按周但不隔 7 天 + assert pd.parse([]) is None and pd.parse([None, "2026-10-01"]) is None + + +def test_calendar_units_and_weekend_days(): + assert pd.calendar_unit(date(2026, 9, 1), date(2026, 9, 30)) == "month" + assert pd.calendar_unit(date(2026, 7, 1), date(2026, 9, 30)) == "quarter" + assert pd.calendar_unit(date(2026, 1, 1), date(2026, 12, 31)) == "year" + assert pd.calendar_unit(date(2026, 9, 2), date(2026, 9, 30)) is None + assert pd.weekend_days(date(2026, 10, 12), date(2026, 10, 18)) == 2 # 周一到周日 + assert pd.weekend_days(date(2026, 10, 16), date(2026, 10, 18)) == 2 # 周五到周日 + assert pd.weekend_days(date(2026, 10, 12), date(2026, 10, 14)) == 0 + assert pd.weekend_days(date(2026, 10, 1), date(2026, 10, 31)) == 9 From 7d31922d00ac018fb62fa529f9664285b3c68372 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 13:04:58 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E4=BA=94=E4=B8=AA=E5=88=86=E6=9E=90?= =?UTF-8?q?=E7=AE=97=E5=AD=90=E4=B8=8E=20run=20/=20recompute=EF=BC=88?= =?UTF-8?q?=E7=BB=86=E6=A1=88=2001=20=C2=A71=E2=80=93=C2=A78=EF=BC=89?= =?UTF-8?q?=EF=BC=9Adecompose=EF=BC=88additive=20=E6=8C=89=E7=BB=B4?= =?UTF-8?q?=E5=BA=A6=E7=AE=97=E8=B4=A1=E7=8C=AE=E3=80=81headline=20?= =?UTF-8?q?=E4=B8=8E=E4=B8=8B=E9=92=BB=E9=97=AD=E5=90=88=E3=80=81=E6=8A=B5?= =?UTF-8?q?=E6=B6=88=E3=80=81=E6=96=B0=E5=87=BA=E7=8E=B0=20/=20=E6=B6=88?= =?UTF-8?q?=E5=A4=B1=EF=BC=9Bratio=20=E4=B8=AD=E7=82=B9=E5=AF=B9=E7=A7=B0?= =?UTF-8?q?=E6=8B=86=E7=BB=93=E6=9E=84=E4=B8=8E=E8=87=AA=E8=BA=AB=E6=95=88?= =?UTF-8?q?=E5=BA=94=E3=80=81=E8=BE=9B=E6=99=AE=E6=A3=AE=E3=80=81=E5=88=86?= =?UTF-8?q?=E6=AF=8D=E5=B0=8F=EF=BC=9Bfactor=20LMDI-I=EF=BC=9B=E4=B8=A4?= =?UTF-8?q?=E4=BB=BD=E5=BF=AB=E7=85=A7=E5=85=88=E5=81=9A=E5=8F=AF=E6=AF=94?= =?UTF-8?q?=E6=80=A7=EF=BC=8C=E5=8F=A3=E5=BE=84=E7=BB=93=E6=9E=84=E4=B8=8D?= =?UTF-8?q?=E5=90=8C=E4=B8=8D=E6=8B=86=EF=BC=89=E3=80=81baseline=EF=BC=88?= =?UTF-8?q?=E4=B8=AD=E4=BD=8D=E6=95=B0=E4=B8=8E=20MAD=20=E7=9A=84=E7=A8=B3?= =?UTF-8?q?=E5=81=A5=20z=E3=80=81=CF=83=CC=82=20=3D=200=E3=80=81=E6=97=A5?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=20lag=207=E3=80=81=E6=96=B0=E9=AB=98?= =?UTF-8?q?=E6=96=B0=E4=BD=8E=EF=BC=89=E3=80=81trend=EF=BC=88Theil?= =?UTF-8?q?=E2=80=93Sen=20+=20Mann=E2=80=93Kendall=E3=80=81=E6=9C=AB?= =?UTF-8?q?=E5=B0=BE=E8=BF=9E=E7=BB=AD=E6=9C=9F=E6=95=B0=EF=BC=89=E3=80=81?= =?UTF-8?q?differ=EF=BC=88rate=20=E7=9A=84=20z=20=E6=A3=80=E9=AA=8C=20/=20?= =?UTF-8?q?Fisher=E3=80=81mean=20=E4=B8=8E=20rows=20=E7=9A=84=20Welch?= =?UTF-8?q?=E3=80=81Holm=E3=80=81rows=20=E7=9A=84=E6=95=8F=E6=84=9F?= =?UTF-8?q?=E6=80=A7=EF=BC=89=E3=80=81comparable=EF=BC=88=E5=90=8C?= =?UTF-8?q?=E5=8F=A3=E5=BE=84=E3=80=81=E7=AA=97=E5=8F=A3=E9=95=BF=E5=BA=A6?= =?UTF-8?q?=E3=80=81=E6=95=B4=E6=9C=9F=E3=80=81=E6=98=9F=E6=9C=9F=E6=9E=84?= =?UTF-8?q?=E6=88=90=E3=80=81=E6=95=B0=E6=8D=AE=E9=97=AE=E9=A2=98=EF=BC=8C?= =?UTF-8?q?=E7=8E=B0=E6=9F=A5=E8=AE=B0=E8=BF=9B=20probes=EF=BC=89=EF=BC=9B?= =?UTF-8?q?run=20=E5=85=88=E7=99=BB=E8=AE=B0=E7=8E=B0=E6=9F=A5=E3=80=81?= =?UTF-8?q?=E6=9F=A5=E5=AE=8C=E5=86=8D=E7=AE=97=EF=BC=8C=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E5=BD=93=E5=A4=A9=E5=92=8C=E7=9B=AE=E5=BD=95=E4=BA=8B=E5=AE=9E?= =?UTF-8?q?=E8=AE=B0=E8=BF=9B=20params=EF=BC=8Crecompute=20=E5=8F=AA?= =?UTF-8?q?=E7=94=A8=E8=AE=B0=E4=B8=8B=E7=9A=84=E3=80=81=E4=B8=8D=E7=8E=B0?= =?UTF-8?q?=E6=9F=A5=EF=BC=9B=E5=88=A4=E5=AE=9A=E7=A0=81=E3=80=81=E5=8E=9F?= =?UTF-8?q?=E5=9B=A0=E7=A0=81=E3=80=81=E9=99=84=E5=B8=A6=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E7=A0=81=E9=83=BD=E5=9C=A8=E5=A5=91=E7=BA=A6=E9=87=8C=EF=BC=9B?= =?UTF-8?q?=E8=BE=93=E5=85=A5=E8=AE=B0=E4=B8=8B=E6=96=B9=E8=A8=80=E5=92=8C?= =?UTF-8?q?=E9=81=AE=E7=BD=A9=E5=88=97=E5=90=8D=EF=BC=8C=E5=A4=8D=E7=AE=97?= =?UTF-8?q?=E9=80=90=E5=AD=97=E6=AE=B5=E7=9B=B8=E5=90=8C=EF=BC=9B=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E7=94=A8=E8=99=9A=E6=9E=84=E9=9B=B6=E5=94=AE=E5=BA=93?= =?UTF-8?q?=EF=BC=8C=E6=AF=8F=E4=B8=AA=E7=94=A8=E4=BE=8B=E9=83=BD=E6=A0=B8?= =?UTF-8?q?=E5=AF=B9=E5=A4=8D=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/__init__.py | 150 ++++- backend/app/analysis/common.py | 136 ++++ backend/app/analysis/comparable.py | 540 ++++++++++++++++ backend/app/analysis/contract.py | 1 + backend/app/analysis/decompose.py | 732 ++++++++++++++++++++++ backend/app/analysis/differ.py | 372 +++++++++++ backend/app/analysis/inputs.py | 2 + backend/app/analysis/series.py | 304 +++++++++ backend/tests/analysis_samples.py | 127 ++++ backend/tests/test_analysis_comparable.py | 182 ++++++ backend/tests/test_analysis_decompose.py | 420 +++++++++++++ backend/tests/test_analysis_differ.py | 138 ++++ backend/tests/test_analysis_probe.py | 129 ++++ backend/tests/test_analysis_run.py | 152 +++++ backend/tests/test_analysis_series.py | 210 +++++++ 15 files changed, 3589 insertions(+), 6 deletions(-) create mode 100644 backend/app/analysis/common.py create mode 100644 backend/app/analysis/comparable.py create mode 100644 backend/app/analysis/decompose.py create mode 100644 backend/app/analysis/differ.py create mode 100644 backend/app/analysis/series.py create mode 100644 backend/tests/analysis_samples.py create mode 100644 backend/tests/test_analysis_comparable.py create mode 100644 backend/tests/test_analysis_decompose.py create mode 100644 backend/tests/test_analysis_differ.py create mode 100644 backend/tests/test_analysis_probe.py create mode 100644 backend/tests/test_analysis_run.py create mode 100644 backend/tests/test_analysis_series.py diff --git a/backend/app/analysis/__init__.py b/backend/app/analysis/__init__.py index a4798d73..67c4ec37 100644 --- a/backend/app/analysis/__init__.py +++ b/backend/app/analysis/__init__.py @@ -8,14 +8,24 @@ - recompute：按结论里记下的 params、settings、probes 和输入快照（按 sha256 取回）重算一遍，不现查；和封存的用 contract.same 比。 -契约（名字、形状、缺省阈值）在 contract.py。 +契约（名字、形状、缺省阈值）在 contract.py。各算子在 decompose.py（变化拆解）、series.py（波动基线、趋势）、differ.py +（差异检验）、comparable.py（可比性检查）；统计在 stats.py，判定语和数的写法在 phrases.py，现查在 probe.py。 + +**一份结论只由 inputs（快照里的格）、probes、params、settings、method 决定**：run 时从 context 读到的东西（运行当天、目录 +里的度量类型和叫法、现查的范围）都记进结论——运行当天、目录事实在 params（today、catalog），现查结果在 probes——复算只读 +记下的。同样的输入两次 run 逐字节相同（现查结果相同时）；recompute 和 run 逐字段相同。 + +**参数**（AnalysisRequest.params；列名不分大小写，按快照认）见各算子文件头；params 里的 catalog、today、depth 由系统填， +调用方写了也不收。 """ from __future__ import annotations from collections.abc import Callable, Mapping +from datetime import date, datetime from typing import Any -from app.analysis.contract import AnalysisRequest, AnalysisResult, Context, InputSnapshot +from app.analysis.contract import (MODES, OPS, AnalysisRequest, AnalysisResult, CallError, Context, InputSnapshot, + effective_settings) #: 按调用里写的名字取回输入快照；没有这个名字返回 None Resolver = Callable[[str], InputSnapshot | None] @@ -23,16 +33,144 @@ Resolver = Callable[[str], InputSnapshot | None] ParentResolver = Callable[[str], AnalysisResult | None] +def _op_mode(request: Mapping[str, Any]) -> tuple[str, str | None]: + from app.analysis.phrases import OP_NAMES + + op = request.get("op") + if op not in OPS: + raise CallError("unknown_op", f"不认识的分析：{op}。能做的有{'、'.join(OP_NAMES[o] for o in OPS)}") + modes = MODES.get(op) + mode = request.get("mode") + if modes is None: + return op, None + if mode not in modes: + raise CallError("unknown_mode", f"{OP_NAMES[op]}要写模式：{' / '.join(modes)}") + return op, mode + + +def _compute(op: str, mode: str | None, raw: Mapping[str, Any], env: Any) -> AnalysisResult: + from app.analysis import comparable, decompose, differ, series + + if op == "decompose": + return {"additive": decompose.additive, "ratio": decompose.ratio, "factor": decompose.factor}[mode](raw, env) # type: ignore[index] + if op == "baseline": + return series.baseline(raw, env) + if op == "trend": + return series.trend(raw, env) + if op == "differ": + return differ.differ(raw, env) + return comparable.comparable(raw, env) + + +def _env(op: str, mode: str | None, settings: Mapping[str, Any], inputs: Any, facts: Any, book: Any, + today: date | None, parents: ParentResolver | None, hypothesis: Any) -> Any: + from app.analysis.common import Env + + return Env(op=op, mode=mode, settings=settings, inputs=inputs, facts=facts, book=book, today=today, + parents=parents, hypothesis=hypothesis if isinstance(hypothesis, str) else None) + + async def run(request: AnalysisRequest, resolve: Resolver, *, settings: Mapping[str, Any], context: Context, parents: ParentResolver | None = None) -> AnalysisResult: """算一份分析结论。settings 是 contract.effective_settings 算好的阈值。""" - raise NotImplementedError("analysis.run：第 1 波 W1-a 实现") + from app.analysis.common import DERIVED_PARAMS + from app.analysis.facts import Facts + from app.analysis.inputs import Inputs + from app.analysis.probe import MAX_ROUNDS, Book, fetch + + op, mode = _op_mode(request) + raw = {k: v for k, v in (request.get("params") or {}).items() if k not in DERIVED_PARAMS} + full = effective_settings(settings) + prober = context.get("probe") + today = context.get("today") or date.today() + if isinstance(today, datetime): + today = today.date() + + def dialect(name: str, snap: Mapping[str, Any]) -> str | None: + given = snap.get("dialect") + if isinstance(given, str) and given: + return given + lookup = getattr(prober, "dialect", None) + return lookup(str(snap.get("source") or "")) if lookup is not None else None + + known: dict[tuple[str, ...], Any] = {} + for rounds in range(MAX_ROUNDS + 1): + book = Book(known) + env = _env(op, mode, full, Inputs(resolve, dialect), Facts(catalogs=context.get("catalogs")), book, today, + parents, request.get("hypothesis")) + result = _compute(op, mode, raw, env) + missing = book.missing + if not missing or rounds == MAX_ROUNDS: + return result + known.update(await fetch(book, context)) + return result # pragma: no cover - 循环里一定返回 def recompute(result: AnalysisResult, load: Callable[[str], InputSnapshot | None], *, load_result: Callable[[str], AnalysisResult | None] | None = None) -> AnalysisResult: - """复算：load 按 sha256 取回输入快照，load_result 按 sha256 取回 parent 那份结论。""" - raise NotImplementedError("analysis.recompute：第 1 波 W1-a 实现") + """复算：load 按 sha256 取回输入快照，load_result 按 sha256 取回 parent 那份结论。用结论里记下的 params（含运行当天、 + 目录事实）、settings、probes，不现查、不读目录。取不回输入时抛 CallError（code input_missing）；方法升版后调用不再成立 + 时同样抛 CallError（照 run 的说法）。和封存的那份用 contract.same 比。""" + from app.analysis.facts import Facts + from app.analysis.inputs import _CELL, Inputs + from app.analysis.probe import Book + + op, mode = _op_mode(result) + params = dict(result.get("params") or {}) + settings = effective_settings(result.get("settings")) + names: dict[str, tuple[str | None, str | None, list[str]]] = {} + parents_at: dict[str, str] = {} + for item in result.get("inputs") or []: + ref = str(item.get("ref") or "") + if item.get("role") == "parent": + parents_at[ref] = str(item.get("artifact") or "") + continue + cell = _CELL.match(ref) if "row" in item else None + name = cell.group("name") if cell else ref + names.setdefault(name, (item.get("artifact"), item.get("dialect"), list(item.get("masked") or []))) + + def resolve(name: str) -> InputSnapshot | None: + artifact, _, masked = names.get(name, (None, None, [])) + found = load(artifact) if artifact else None + if not isinstance(found, Mapping): + raise CallError("input_missing", f"复算取不回 {name} 的查询快照") + return {**found, "artifact": artifact, "masked": masked} # type: ignore[typeddict-item] + + def dialect(name: str, snap: Mapping[str, Any]) -> str | None: + return names.get(name, (None, None, []))[1] + + def parents(name: str) -> AnalysisResult | None: + artifact = parents_at.get(name) + found = load_result(artifact) if artifact and load_result is not None else None + if not isinstance(found, Mapping): + raise CallError("input_missing", f"复算取不回上一层的分析结论 {name}") + return found # type: ignore[return-value] + + today = date.fromisoformat(params["today"]) if isinstance(params.get("today"), str) else None + env = _env(op, mode, settings, Inputs(resolve, dialect), Facts(recorded=params.get("catalog") or {}), + Book(result.get("probes") or []), today, parents, result.get("hypothesis")) + raw = {k: v for k, v in params.items() if k not in ("catalog", "today", "depth")} + return _compute(op, mode, raw, env) -__all__ = ["ParentResolver", "Resolver", "recompute", "run"] +def exhausted(settings: Mapping[str, Any]) -> CallError: + """本运行分析次数用完（max_calls）的调用错误：入口数着次数，到了上限交回它，不再算。""" + limit = int(effective_settings(settings)["max_calls"]) + return CallError("max_calls", f"这次运行的分析次数已经用完（最多 {limit} 次）：用已有的分析结论写，不再新做分析") + + +def make_prober(*args: Any, **kwargs: Any) -> Any: + """见 app/analysis/probe.make_prober。""" + from app.analysis.probe import make_prober as make + + return make(*args, **kwargs) + + +def comparable_reasons(result: Mapping[str, Any]) -> list[str]: + """可比性结论里 fail、warn 的原因，一句一个（口径卡指标的 comparability.reasons）。""" + from app.analysis.comparable import reasons + + return reasons(result) + + +__all__ = ["ParentResolver", "Resolver", "comparable_reasons", "exhausted", "make_prober", "recompute", "run"] diff --git a/backend/app/analysis/common.py b/backend/app/analysis/common.py new file mode 100644 index 00000000..bb323774 --- /dev/null +++ b/backend/app/analysis/common.py @@ -0,0 +1,136 @@ +"""算子共用的：一次计算的环境（输入、目录事实、现查簿、阈值）、结论的组装、几样参数的规范化。 + +每个算子是 (raw params, Env) → AnalysisResult 的纯函数：只经 Env 取输入快照（Inputs）、目录事实（Facts）、现查结果 +（Book），阈值只来自 Env.settings。run 和 recompute 只是装 Env 的方式不同（见 app/analysis/__init__.py）。 +""" +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import date +from fractions import Fraction +from typing import Any + +from app.analysis import phrases as ph +from app.analysis.contract import METHODS, AnalysisResult, CallError +from app.analysis.facts import Facts +from app.analysis.inputs import Inputs +from app.analysis.numbers import to_fraction +from app.analysis.probe import Book + +#: 结论的 params 里由系统填、不收调用方的几项：目录事实、运行当天、下钻的层数 +DERIVED_PARAMS = frozenset({"catalog", "today", "depth"}) + + +@dataclass +class Env: + op: str + mode: str | None + settings: Mapping[str, Any] + inputs: Inputs + facts: Facts + book: Book + today: date | None + parents: Callable[[str], Mapping[str, Any] | None] | None + hypothesis: str | None + + def threshold(self, key: str) -> Fraction: + value = to_fraction(self.settings[key]) + assert value is not None, key + return value + + +def label_for(env: Env, raw: Any, snap: Any, column: str | None, *, default: str | None = None) -> str: + """判定语里的度量叫法：agent 给的 label → 目录里的中文名 → default → 列名，依次取第一个消毒后不为空的（agent 给了能用的 + 就不去查目录）。""" + if ph.clean_label(raw): + return ph.clean_label(raw) + catalog = env.facts.label(snap, column) if column else None + return ph.pick_label(catalog, default, column) + + +def column_label(env: Env, snap: Any, column: str) -> str: + """维度、因子这类列的叫法：目录里的中文名，没有就是列名（都消毒）。""" + return ph.pick_label(env.facts.label(snap, column), column) + + +def today_param(env: Env) -> str: + """要判本期完不完整的算子把运行当天记进 params（复算用记下的那一天）。""" + day = env.today or date.today() + env.today = day + return day.isoformat() + + +def build(env: Env, params: dict[str, Any], used: Sequence[str], *, status: str = "ok", + reason_code: str | None = None, reason: str | None = None, table: Mapping[str, Any] | None = None, + values: Mapping[str, Any] | None = None, verdict: Mapping[str, Any] | None = None, criterion: str, + phrase: str, caveats: Sequence[Mapping[str, str]] = ()) -> AnalysisResult: + """组装一份结论：形状严格按 contract.AnalysisResult，params 补上查过的目录事实。""" + record = env.facts.record() + out_params = {k: v for k, v in params.items() if v is not None} + if record: + out_params["catalog"] = record + return { + "op": env.op, + "mode": env.mode, + "method": METHODS[(env.op, env.mode)], + "params": out_params, + "settings": {k: env.settings[k] for k in used}, + "inputs": list(env.inputs.records), + "probes": env.book.used, + "status": status, # type: ignore[typeddict-item] + "reason_code": reason_code, + "reason": reason, + "table": dict(table) if table is not None else None, # type: ignore[typeddict-item] + "values": dict(values or {}), + "verdict": dict(verdict) if verdict is not None else None, # type: ignore[typeddict-item] + "criterion": criterion, + "phrase": phrase, + "caveats": [dict(c) for c in caveats], # type: ignore[misc] + "hypothesis": env.hypothesis, + } + + +def refuse(env: Env, params: dict[str, Any], used: Sequence[str], code: str, reason: str, *, + criterion: str, caveats: Sequence[Mapping[str, str]] = ()) -> AnalysisResult: + """算不了（status refused）：判定语是「{算子的叫法}没有做成：{原因}」。""" + return build(env, params, used, status="refused", reason_code=code, reason=reason, criterion=criterion, + phrase=ph.refused(env.op, reason), caveats=caveats) + + +def need(raw: Mapping[str, Any], key: str, what: str) -> Any: + """必填的参数；没写是调用错误。""" + value = raw.get(key) + if value is None or (isinstance(value, str) and not value.strip()) or value == []: + raise CallError("missing_param", f"没写{what}") + return value + + +def written_places(raw: Any) -> int: + """快照里一格写到第几位小数（headline 是取整过的，按它的书写精度容差）：数字文本按字面，浮点按最短写法，整数为 0。""" + if isinstance(raw, bool) or raw is None: + return 0 + text = repr(raw) if isinstance(raw, float) else str(raw).strip() + if "e" in text.lower(): + return 0 + return len(text.split(".", 1)[1]) if "." in text else 0 + + +def closes(total: Fraction, cell: Fraction, raw: Any) -> bool: + """各段之和（或合起来的比率）和总数那一格对得上：十进制逐位相等；总数取整过的，差在它书写精度的半个末位以内也算；浮点 + 算出来的差在最后几位的也算（相对误差 1e-9 以内，同 engine/compare._equal）。""" + if total == cell: + return True + gap = abs(total - cell) + if gap <= Fraction(1, 2) / Fraction(10) ** written_places(raw): + return True + scale = max(abs(total), abs(cell)) + return scale != 0 and gap <= scale * Fraction(1, 10 ** 9) + + +def sign_word(value: Fraction | float | int) -> str | None: + return "up" if value > 0 else "down" if value < 0 else None + + +__all__ = ["DERIVED_PARAMS", "Env", "build", "closes", "column_label", "label_for", "need", "refuse", "sign_word", + "today_param", "written_places"] diff --git a/backend/app/analysis/comparable.py b/backend/app/analysis/comparable.py new file mode 100644 index 00000000..c34abfce --- /dev/null +++ b/backend/app/analysis/comparable.py @@ -0,0 +1,540 @@ +"""可比性检查（细案 01 §6）：比两期之前先确认能比。三处用到：agent 单独调用（analysis_comparable）；变化拆解给了两份快照时 +先做；口径卡跨两份查询算环比、同比时自动做（入口经 run 调）。波动基线、趋势判本期完整用这里的 span_coverage。 + +五项检查，每项 pass / warn / fail / unknown（unknown 只写「核对不了」，不判不可比）： + +- 同口径：两份 SQL 一字不差 → pass；规范化（data/sql_semantics.normalize_sql）后只差字面量：把时间条件的界（data/coverage + 认出的时间谓词的界，整个表达式）和像日期的字面量换成同一个占位再比，一样就是只差日期 → pass，不一样就是比的两段不是 + 两期 → fail「两边除了日期，筛选的还不一样」；规范形式不同交给口径账本的 explain_sql_diff，等价 → pass，只差时间范围、排序 + → pass，否则 fail，写出差在哪些方面（只写方面，不写 SQL 原文里的值）。两份来自不同的数据源 fail。同一份快照里按期次分组 + 的天然同口径。 +- 窗口长度：用 data/coverage 的 filters 取业务日期列（目录的 business_date；没写就取唯一那一列时间谓词）上的区间，界是字面 + 量的直接解析，按运行当天现算的在库里算（Book 的 bound，记进 probes）；算两边天数。整月对整月、整季对整季算一样长。 + 不相等：度量是流量 fail，比率、存量、没标的 warn。按期次分组的：两期粒度相同，一样长。 +- 整期：窗口（或这一期）的终点和表里这一列的实际最大值（Book 的 range：现查，查不成退回剖析事实）、运行当天比；数据从 + 窗口起点之后才开始的也算不完整。不完整：流量 fail（「本期数据只到 2026-10-15，还差 3 天」），其余 warn；表的范围查不到时 + 只能按运行当天判，判不出的 unknown。 +- 星期构成：按天的窗口、天数不是 7 的倍数时数两边的周六、周日天数，不同 warn（「两段的周末天数不同：4 天对 6 天」）。整月、 + 整季、按周的不看。 +- 数据问题：两边查询结果上「查询区间里这张表没有数据」（快照的 no_data）fail，SQL 检查的错误级结果（快照的 checks，level + error）warn。 + +判定：有 fail → incomparable；只有 warn → conditional；其余 comparable（有核对不了的项照实写在判定语后面）。 +""" +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from datetime import date, timedelta +from typing import Any + +from app.analysis import periods as pd +from app.analysis import phrases as ph +from app.analysis.facts import Facts, date_source +from app.analysis.inputs import Snap +from app.analysis.probe import Book +from app.data.names import name_key + +CHECKS = ("same_query", "window", "complete", "weekdays", "data") +CHECK_NAMES = {"same_query": "同口径", "window": "窗口长度", "complete": "整期", "weekdays": "星期构成", + "data": "数据问题"} +STATUS_NAMES = {"pass": "通过", "warn": "提醒", "fail": "不通过", "unknown": "核对不了"} +#: explain_sql_diff 的方面 → 叫法 +_ASPECTS = {"time": "时间列", "time_range": "时间范围", "filter": "筛选条件", "aggregate": "聚合方式", + "expression": "选出来的列", "join": "关联", "tables": "查的表", "set_op": "合并方式", "group_by": "分组", + "having": "分组后的筛选", "limit": "取哪几行", "order": "排序", "parse": "写法", "other": "写法"} +#: 只差这几个方面的两份 SQL 仍是同一个量的两期 +_SAME_ASPECTS = frozenset({"time_range", "order"}) +#: 一项 SQL 检查的说明最多引多少字 +_CHECK_TEXT_MAX = 80 + + +@dataclass +class Check: + id: str + status: str = "unknown" + reasons: list[str] = field(default_factory=list) + base: str | None = None + target: str | None = None + + +@dataclass +class Span: + """一边的时间范围：start / end 是窗口（或这一期）的头尾（含）；table / column 是这段时间按的那一列（探查表的范围用）； + sql_window 是 SQL 时间条件在这一列上的区间（按期次分组时看它有没有截掉这一期）。""" + + label: str + snap: Snap + start: date | None = None + end: date | None = None + table: str | None = None + column: str | None = None + sql_window: Any = None + unit: str | None = None # 这一段是不是整月、整季、整年（calendar_unit），按期次时是粒度 + + +@dataclass +class Outcome: + checks: list[Check] + verdict: str + reasons: list[str] + unknown: list[str] + phrase: str + structure: str | None # 同口径不通过的原因（变化拆解据此不拆） + base: Span | None = None + target: Span | None = None + + def check(self, cid: str) -> Check: + return next(c for c in self.checks if c.id == cid) + + +# -------------------------------------------------------------------------- +# 同口径 +# -------------------------------------------------------------------------- + + +def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None: + """时间条件的界、像日期的字面量换成同一个占位之后的规范形式（文本 + 剩下的字面量）。解析不了为 None。""" + from sqlglot import exp + + from app.data import coverage + from app.data.sql_semantics import normalize_sql + + tree = coverage.parse_sql(sql, read) + if tree is None: + return None + for flt in coverage.filters(tree): + for _, node, _ in flt.conds: + if node.parent is not None: + node.replace(exp.Literal.string("__day__")) + for lit in list(tree.find_all(exp.Literal)): + if coverage.literal_bound(lit)[1] is not None: + lit.replace(exp.Literal.string("__day__")) + normalized = normalize_sql(tree.sql(dialect=read), read) + if normalized.parse_error: + return None + return normalized.text, normalized.literals + + +def same_query(base: Snap, target: Snap) -> Check: + from app.data.sql_semantics import explain_sql_diff, normalize_sql + + check = Check("same_query") + if base.source != target.source: + check.status, check.reasons = "fail", ["两份查询来自不同的数据源"] + return check + a, b = base.sql.strip(), target.sql.strip() + if a == b: + check.status = "pass" + return check + read = base.dialect or target.dialect + na, nb = normalize_sql(a, read), normalize_sql(b, read) + if na.parse_error or nb.parse_error: + check.reasons = ["SQL 解析不了"] + return check + if na.text == nb.text: + ua, ub = _undated(a, read), _undated(b, read) + if ua is None or ub is None: + return check + if ua == ub: + check.status = "pass" + else: + check.status, check.reasons = "fail", ["两边除了日期，筛选的还不一样"] + return check + try: + diff = explain_sql_diff(a, b, read) + except Exception: # noqa: BLE001 - 比不了就当核对不了 + return check + aspects = [d.aspect for d in diff.items] + if diff.equivalent or (aspects and set(aspects) <= _SAME_ASPECTS): + check.status = "pass" + return check + names = list(dict.fromkeys(_ASPECTS.get(x, "写法") for x in aspects if x not in _SAME_ASPECTS)) or ["写法"] + check.status, check.reasons = "fail", [f"两份查询的口径结构不同（差在{'、'.join(names)}）"] + return check + + +# -------------------------------------------------------------------------- +# 窗口 +# -------------------------------------------------------------------------- + + +def _bound_days(snap: Snap, conds: Sequence[tuple[str, Any, bool]], book: Book) -> dict[int, Any]: + from app.data import coverage + + days: dict[int, Any] = {} + for _, node, _ in conds: + literal, day = coverage.literal_bound(node) + if literal: + days[id(node)] = day + continue + try: + text = node.sql(dialect=snap.dialect) + except Exception: # noqa: BLE001 + days[id(node)] = None + continue + probe = book.bound(snap.source, text) + days[id(node)] = coverage.as_day(probe.get("value")) if probe else None + return days + + +def _filters_on(snap: Snap, measure: str | None, facts: Facts, + column: tuple[str, str] | None = None) -> tuple[Any, str | None]: + """SQL（收窄到度量那一项）里按业务日期列（或唯一那一列时间谓词，或给定的那一列）的时间条件：(那段条件, 认不出的原因)。""" + from app.data import coverage + + tree = coverage.parse_sql(snap.sql, snap.dialect) + if tree is None: + return None, "SQL 解析不了" + narrowed = coverage.narrowed(tree, measure, snap.dialect) if measure else tree + flts = coverage.filters(narrowed) + groups: dict[tuple[str, str], list[Any]] = {} + for flt in flts: + groups.setdefault((name_key(flt.table_name), name_key(flt.column_name)), []).append(flt) + if column is not None: + wanted = (name_key(column[0]), name_key(column[1])) + wanted_short = (name_key(column[0].rsplit(".", 1)[-1]), wanted[1]) + chosen = groups.get(wanted) or next((v for k, v in groups.items() + if (k[0].rsplit(".", 1)[-1], k[1]) == wanted_short), None) + if chosen is None: + return None, None + else: + if not groups: + return None, "SQL 里没有时间条件" + dated = [v for v in groups.values() + if (bd := facts.business_date(snap.source, v[0].table_name)) and name_key(bd) == name_key(v[0].column_name)] + if len(dated) == 1: + chosen = dated[0] + elif len(groups) == 1: + chosen = next(iter(groups.values())) + else: + return None, "时间条件不止一列，认不出按哪一列算" + if len({f.key for f in chosen}) > 1: + return None, "这一列上有好几段时间条件" + return chosen[0], None + + +def sql_span(label: str, snap: Snap, measure: str | None, facts: Facts, book: Book) -> tuple[Span, str | None]: + """一份查询的时间窗口（两份快照比两期时）。""" + from app.data import coverage + + span = Span(label, snap) + flt, why = _filters_on(snap, measure, facts) + if flt is None: + return span, why + if flt.column_name.lower() in snap.masked: + return span, "时间列是遮罩列" # 遮罩列不探表的范围 + span.table, span.column = flt.table_name, flt.column_name + window = coverage.window_of(flt.conds, _bound_days(snap, flt.conds, book)) + if window is None or window.start is None or window.end is None: + return span, "时间窗口没有两头" + span.start, span.end = window.start, window.end + span.unit = pd.calendar_unit(window.start, window.end) + return span, None + + +def period_span(label: str, snap: Snap, period: Any, item: pd.Period, grain: str, measure: str | None, + facts: Facts, book: Book) -> Span: + """按期次分组的一份快照里的一期：这一期覆盖的日子，加上 SQL 在期次那一列上的时间条件。""" + from app.data import coverage + + span = Span(label, snap, item.start, item.end, unit=grain) + found = date_source(snap.sql, period) + if found is None: + return span + span.table, span.column = found + if span.column.lower() in snap.masked: + span.table = span.column = None + return span + flt, _ = _filters_on(snap, measure, facts, column=found) + if flt is not None: + span.sql_window = coverage.window_of(flt.conds, _bound_days(snap, flt.conds, book)) + return span + + +def _range(span: Span, book: Book) -> tuple[bool, date | None, date | None, bool]: + """(查得到, 最早, 最晚, 一个值都没有)：表里这一列的实际范围。""" + from app.data import coverage + + if not (span.table and span.column): + return False, None, None, False + probe = book.range(span.snap.source, span.table, span.column) + if probe is None: + return False, None, None, False + lo = coverage.as_day(probe.get("min")) if probe.get("min") else None + hi = coverage.as_day(probe.get("max")) if probe.get("max") else None + empty = probe.get("basis") == "live" and probe.get("min") is None and probe.get("max") is None + return True, lo.day if lo else None, hi.day if hi else None, empty + + +def span_coverage(span: Span, today: date, book: Book) -> tuple[str, str | None]: + """这一段（窗口或一期）的数据齐不齐：(pass / short / unknown, 不齐的那一句)。没过完（终点不早于运行当天：当天的数据还没 + 到齐）、SQL 的时间条件截掉了一部分、表里的数据没到（最晚早于终点）或来得晚（最早晚于起点），都算不齐。""" + if span.start is None or span.end is None: + return "unknown", None + lo, hi = span.start, span.end + window = span.sql_window + if window is not None: + if window.start is not None: + lo = max(lo, window.start) + if window.end is not None: + hi = min(hi, window.end) + if span.end >= today: + hi = min(hi, today - timedelta(days=1)) + known, dmin, dmax, empty = _range(span, book) + if empty: + return "short", f"{span.label}在 {span.start.isoformat()} 到 {span.end.isoformat()} 没有数据" + if dmin is not None: + lo = max(lo, dmin) + if dmax is not None: + hi = min(hi, dmax) + if hi < lo: + return "short", f"{span.label}在 {span.start.isoformat()} 到 {span.end.isoformat()} 没有数据" + parts = [] + if hi < span.end: + parts.append(f"{span.label}数据只到 {hi.isoformat()}，还差 {(span.end - hi).days} 天") + if lo > span.start: + parts.append(f"{span.label}数据从 {lo.isoformat()} 才开始，缺前 {(lo - span.start).days} 天") + if parts: + return "short", "，".join(parts) + return ("pass", None) if known and dmax is not None else ("unknown", None) + + +def _days(span: Span) -> int | None: + return (span.end - span.start).days + 1 if span.start and span.end else None + + +def _shown(span: Span) -> str | None: + if span.start is None or span.end is None: + return None + if span.start == span.end: + return f"{span.start.isoformat()}（1 天）" + return f"{span.start.isoformat()} 到 {span.end.isoformat()}（{_days(span)} 天）" + + +# -------------------------------------------------------------------------- +# 五项检查和判定 +# -------------------------------------------------------------------------- + + +def _window_check(base: Span, target: Span, flow: bool, *, grouped: bool) -> Check: + check = Check("window", base=_shown(base), target=_shown(target)) + a, b = _days(base), _days(target) + if a is None or b is None: + return check + if a == b or grouped or (base.unit and base.unit == target.unit): + check.status = "pass" + return check + check.status = "fail" if flow else "warn" + check.reasons = [f"两段的天数不同：{a} 天对 {b} 天"] + return check + + +def _complete_check(sides: Sequence[Span], today: date, book: Book, flow: bool) -> Check: + check = Check("complete") + shorts, unknown = [], False + for span in sides: + status, reason = span_coverage(span, today, book) + if status == "short": + shorts.append(reason) + elif status == "unknown": + unknown = True + if span.label == "基期": + check.base = reason or STATUS_NAMES[status if status != "short" else "fail"] + else: + check.target = reason or STATUS_NAMES[status if status != "short" else "fail"] + if shorts: + check.status, check.reasons = ("fail" if flow else "warn"), [r for r in shorts if r] + elif not unknown: + check.status = "pass" + return check + + +def _weekday_check(base: Span, target: Span, *, grain: str | None) -> Check: + check = Check("weekdays") + a, b = _days(base), _days(target) + if a is None or b is None: + return check + if grain in ("week", "month", "quarter", "seq") or (a % 7 == 0 and b % 7 == 0) or \ + (base.unit and base.unit == target.unit and base.unit in ("month", "quarter", "year")): + check.status = "pass" + return check + wa, wb = pd.weekend_days(base.start, base.end), pd.weekend_days(target.start, target.end) # type: ignore[arg-type] + check.base, check.target = f"周末 {wa} 天", f"周末 {wb} 天" + if wa == wb: + check.status = "pass" + else: + check.status, check.reasons = "warn", [f"两段的周末天数不同：{wa} 天对 {wb} 天"] + return check + + +def _data_check(sides: Sequence[tuple[str, Snap]], measure: str | None) -> Check: + from app.data import coverage + + check = Check("data", status="pass") + fails, warns = [], [] + for label, snap in sides: + found = coverage.no_data_of(snap.data, measure) if measure else \ + next(iter(snap.data.get("no_data") or []), None) + if found: + fails.append(f"{label}查询的区间里这张表没有数据") + errors = [c for c in snap.data.get("checks") or [] if isinstance(c, Mapping) and c.get("level") == "error"] + if errors: + text = str(errors[0].get("message") or "").strip().replace("；", "，") + if len(text) > _CHECK_TEXT_MAX: + text = text[:_CHECK_TEXT_MAX - 1] + "…" + warns.append(f"{label}查询的 SQL 检查有错误：{text}" if text else f"{label}查询的 SQL 检查有错误") + if fails: + check.status = "fail" + elif warns: + check.status = "warn" + check.reasons = fails + warns + return check + + +def _decide(checks: list[Check], base: Span | None, target: Span | None) -> Outcome: + fails = [r for c in checks if c.status == "fail" for r in c.reasons] + warns = [r for c in checks if c.status == "warn" for r in c.reasons] + unknown = [CHECK_NAMES[c.id] for c in checks if c.status == "unknown"] + verdict = "incomparable" if fails else "conditional" if warns else "comparable" + reasons = fails + warns + structure = next((c.reasons[0] for c in checks if c.id == "same_query" and c.status == "fail" and c.reasons), None) + return Outcome(checks, verdict, reasons, unknown, ph.comparable_phrase(verdict, reasons, unknown), structure, + base, target) + + +def compare_queries(base: Snap, target: Snap, measure: str | None, facts: Facts, book: Book, today: date) -> Outcome: + """两份快照比两期（本周一份、上周一份）。measure 是两边的度量列（收窄 SQL 找时间条件、认度量类型）；没有为 None。""" + same = same_query(base, target) + b_span, _ = sql_span("基期", base, measure, facts, book) + t_span, _ = sql_span("本期", target, measure, facts, book) + flow = bool(measure) and facts.measure(target, measure)[0] == "flow" # type: ignore[arg-type] + checks = [same, _window_check(b_span, t_span, flow, grouped=False), + _complete_check([b_span, t_span], today, book, flow), _weekday_check(b_span, t_span, grain=None), + _data_check([("基期", base), ("本期", target)], measure)] + return _decide(checks, b_span, t_span) + + +def compare_periods(snap: Snap, period: str, base: pd.Period, target: pd.Period, grain: str, measure: str | None, + facts: Facts, book: Book, today: date) -> Outcome: + """同一份快照里按期次分组的两期：天然同口径；窗口按期次的粒度；整期看这两期各自齐不齐。""" + same = Check("same_query", status="pass") + b_span = period_span("基期", snap, period, base, grain, measure, facts, book) + t_span = period_span("本期", snap, period, target, grain, measure, facts, book) + flow = bool(measure) and facts.measure(snap, measure)[0] == "flow" # type: ignore[arg-type] + if grain == "seq": + checks = [same, Check("window"), Check("complete"), Check("weekdays"), _data_check([("", snap)], measure)] + else: + checks = [same, _window_check(b_span, t_span, flow, grouped=True), + _complete_check([b_span, t_span], today, book, flow), + _weekday_check(b_span, t_span, grain=grain), _data_check([("", snap)], measure)] + return _decide(checks, b_span, t_span) + + +def table_of(outcome: Outcome) -> dict[str, Any]: + """检查表（结论的 table）：检查项、结果、基期、本期、说明。""" + rows = [[CHECK_NAMES[c.id], STATUS_NAMES[c.status], c.base, c.target, "；".join(c.reasons) or None] + for c in outcome.checks] + return {"columns": ["check", "status", "base", "target", "note"], + "labels": {"check": "检查项", "status": "结果", "base": "基期", "target": "本期", "note": "说明"}, + "rows": rows} + + +def values_of(outcome: Outcome) -> dict[str, Any]: + out: dict[str, Any] = {c.id: c.status for c in outcome.checks} + for side, span in (("base", outcome.base), ("target", outcome.target)): + if span is not None and span.start and span.end: + out[f"{side}_start"] = span.start.isoformat() + out[f"{side}_end"] = span.end.isoformat() + out[f"{side}_days"] = str(_days(span)) + out[f"{side}_weekend"] = str(pd.weekend_days(span.start, span.end)) + return out + + +def reasons(result: Mapping[str, Any]) -> list[str]: + """一份可比性结论里 fail、warn 的原因，一句一个（口径卡指标的 comparability.reasons 用）。按检查表的说明拆开。""" + out: list[str] = [] + table = result.get("table") or {} + for row in table.get("rows") or []: + if len(row) >= 5 and row[1] in (STATUS_NAMES["fail"], STATUS_NAMES["warn"]) and row[4]: + out.extend(str(row[4]).split("；")) + return out + + +# -------------------------------------------------------------------------- +# 算子：analysis_comparable +# -------------------------------------------------------------------------- + + +def comparable(raw: Mapping[str, Any], env: Any) -> Any: + """可比性检查这个算子。参数：两份快照 base_query、target_query；或一份快照 query + period（期次列）+ base、target（期次 + 列里的两个值）、可选 grain。都可以写 measure（度量列：收窄 SQL 找时间条件、按目录认它是不是流量；不写时只有一列数值列 + 就用它）。""" + from app.analysis.common import build, need, today_param + from app.analysis.contract import CallError + from app.analysis.inputs import same_value + + params: dict[str, Any] = {} + two = raw.get("base_query") is not None or raw.get("target_query") is not None + if two: + base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号") + target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号") + if base is target: + from app.analysis.decompose import SAME_QUERY + + raise CallError("same_query", SAME_QUERY) + params.update(base_query=base.ref, target_query=target.ref) + else: + base = target = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号") + params["query"] = base.ref + measure = target.column(raw["measure"], "度量列") if raw.get("measure") is not None else _only_number(target) + if measure is not None and two: + base.column(measure, "度量列") + params["measure"] = measure + params["today"] = today_param(env) + if two: + env.inputs.use(base, "base", [measure] if measure else []) + env.inputs.use(target, "target", [measure] if measure else []) + outcome = compare_queries(base, target, measure, env.facts, env.book, env.today) + else: + period = target.column(need(raw, "period", "期次列"), "期次列") + grain = raw.get("grain") + if grain is not None and grain not in (*pd.GRAINS, "seq"): + raise CallError("bad_grain", "粒度只能写 day、week、month、quarter，或者不写（按期次值推断）") + params.update(period=period, base=need(raw, "base", "基期（期次列里的值）"), + target=need(raw, "target", "本期（期次列里的值）")) + i = target.index(period) + values = [row[i] if i < len(row) else None for row in target.rows] + rows = {side: [r for r, v in enumerate(values) if same_value(params[side], v)] for side in ("base", "target")} + for side, what in (("base", "基期"), ("target", "本期")): + if not rows[side]: + raise CallError("period_not_found", f"期次列「{period}」里没有「{params[side]}」：{what}要写 {target.ref} " + "里有的期次值") + env.inputs.use(target, "data", [period, *([measure] if measure else [])], rows["base"] + rows["target"]) + parsed = pd.parse(values, grain) + if parsed is None: + item_b = item_t = pd.Period(None, 0, None, None) + grain = "seq" + else: + items, grain = parsed + item_b, item_t = items[rows["base"][0]], items[rows["target"][0]] + params["grain"] = grain + outcome = compare_periods(target, period, item_b, item_t, grain, measure, env.facts, env.book, env.today) + caveats = [] + window = outcome.check("window") + if measure and window.status == "warn" and env.facts.measure(target, measure)[0] is None: + caveats.append(ph.caveat("measure_unlabeled", ph.unlabeled_text(measure, "窗口不等只作提醒"))) + verdict = {"code": outcome.verdict, "subjects": [], "direction": None} + return build(env, params, [], table=table_of(outcome), values=values_of(outcome), verdict=verdict, + criterion=ph.CRITERION_COMPARABLE, phrase=outcome.phrase, caveats=caveats) + + +def _only_number(snap: Snap) -> str | None: + """没写度量列时：快照里唯一一列（没遮罩的）数值列；没有或不止一列为 None。""" + found = [c for c in snap.columns if c.lower() not in snap.masked and snap.kind(c) == "number"] + return found[0] if len(found) == 1 else None + + +__all__ = ["CHECKS", "CHECK_NAMES", "Check", "Outcome", "STATUS_NAMES", "Span", "comparable", "compare_periods", + "compare_queries", "period_span", "reasons", "same_query", "span_coverage", "sql_span", "table_of", + "values_of"] diff --git a/backend/app/analysis/contract.py b/backend/app/analysis/contract.py index 8e82268d..75b7af52 100644 --- a/backend/app/analysis/contract.py +++ b/backend/app/analysis/contract.py @@ -208,6 +208,7 @@ class AnalysisInput(TypedDict, total=False): column: str value: str | None dialect: str # 读这份快照的 SQL 用的方言（第 1 波 W1-a 加，可选）：复算按它读，界的写法才一样 + masked: list[str] # 这份快照当时不能当维度、度量的列（小写，只记列名；第 1 波 W1-a 加，可选）：复算按它认 class Probe(TypedDict, total=False): diff --git a/backend/app/analysis/decompose.py b/backend/app/analysis/decompose.py new file mode 100644 index 00000000..f55224ce --- /dev/null +++ b/backend/app/analysis/decompose.py @@ -0,0 +1,732 @@ +"""变化拆解（细案 01 §2）：变化从哪来。三种模式，agent 按指标的性质选，算子核对选得对不对。 + +**参数**（AnalysisRequest.params；列名不分大小写，按快照认）： + +- 一份快照：query（编号）、period（期次列）、base / target（期次列里的两个值）；或两份快照：base_query、target_query（各自 + 只有维度列和度量列，先做可比性检查：口径结构不同的不拆，算不了 incomparable_sides；其余不可比的照样算，判定后附上 + 可比性结论）。 +- additive：by（维度列，1–2 列）、measure（度量列）；可选 headline {base, target}（基期、本期总数各一格，如 Q1.r1.gmv，用来 + 核对闭合）、parent {of: A3, key: [华东]}（下钻：这一层各段之和要等于 A3 里华东那一段）。 +- ratio：by、numerator（分子列）、denominator（分母列）；可选 headline {base, target}（整体比率的两格）、parent（上一层的比率 + 拆解）。 +- factor：product（乘积列）、factors（因子列，2–4 个）；一份快照按 period / base / target 取两行，两份快照各一行。 +- 都可以写 label（度量的叫法，消毒后进判定语）。 + +**计算**全程 Fraction（快照里的数按字面换成精确的有理数）：各段之和逐位等于总变化；比率拆解的结构效应、自身效应按中点 +对称拆，Σmix + Σrate 逐位等于比率的变化；乘积拆解是 LMDI-I（对数），浮点，各因子贡献之和在浮点误差内等于总变化。判定、 +阈值、边界照细案 §2.1–§2.4。 +""" +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from fractions import Fraction +from typing import Any + +from app.analysis import comparable as cmp +from app.analysis import phrases as ph +from app.analysis.common import Env, build, closes, column_label, label_for, need, refuse, sign_word, today_param +from app.analysis.contract import CallError +from app.analysis.inputs import Snap, as_list, hashable, same_value +from app.analysis.numbers import canon, to_fraction + +#: 辛普森：分母够大的段里，按平均权重算有这么多的段比率和整体反着变 +SIMPSON_SHARE = Fraction(4, 5) +#: 下钻、总数的写法（调用错误里教 agent 怎么写） +PARENT_SHAPE = "下钻要写成 {of: A3, key: [华东]} 这样：of 是上一层变化拆解的编号，key 是那一段的维度值" +HEADLINE_SHAPE = "总数要写成 {base: Q1.r1.gmv, target: Q1.r0.gmv} 这样：base 是基期总数那一格，target 是本期总数那一格" +SAME_QUERY = "基期和本期是同一份查询：两期各给一份查询，或者给一份按期次分组的查询和期次列" +#: 乘积闭合的最小相对误差 +_FACTOR_REL = Fraction(1, 10 ** 6) +_NOT_ADDITIVE = {"ratio": "比率", "identifier": "标识", "status": "状态", "attribute": "属性"} + + +@dataclass +class Side: + """一期的各段：键 → 值（ratio 模式是 (分子, 分母)）；nulls 是这一期度量格为空的段；rows 是用到的行。""" + + snap: Snap + values: dict[tuple[Any, ...], Any] = field(default_factory=dict) + nulls: list[tuple[Any, ...]] = field(default_factory=list) + rows: list[int] = field(default_factory=list) + + +@dataclass +class Segment: + key: tuple[Any, ...] + base: Any + target: Any + flags: list[str] = field(default_factory=list) + + +def _key_strings(key: Sequence[Any]) -> list[str]: + return [ph.key_text([k]) for k in key] + + +def _sides(raw: Mapping[str, Any], env: Env, roles: Sequence[tuple[str, str]], *, by_needed: bool = True + ) -> tuple[dict[str, Any], Snap, Snap, dict[str, str], list[str]]: + """一份或两份快照：(规范化的参数, 基期快照, 本期快照, {角色: 列名}, 维度列)。roles 是 [(参数名, 角色的叫法)]。""" + params: dict[str, Any] = {} + two = raw.get("base_query") is not None or raw.get("target_query") is not None + if two: + base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号") + target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号") + if base is target: + raise CallError("same_query", SAME_QUERY) + params.update(base_query=base.ref, target_query=target.ref) + else: + base = target = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号") + params["query"] = base.ref + by: list[str] = [] + if by_needed: + wanted = as_list(raw.get("by")) + if not 1 <= len(wanted) <= 2: + raise CallError("bad_by", "维度列要写 1 到 2 列") + by = [target.column(c, "维度列") for c in wanted] + if two: + for c in by: + base.column(c, "维度列") + params["by"] = by + cols: dict[str, str] = {} + for key, what in roles: + if key == "factors": + continue + col = target.column(need(raw, key, what), what) + if two: + base.column(col, what) + cols[key] = col + params[key] = col + if not two: + params["period"] = target.column(need(raw, "period", "期次列"), "期次列") + params["base"] = need(raw, "base", "基期（期次列里的值）") + params["target"] = need(raw, "target", "本期（期次列里的值）") + if same_value(params["base"], params["target"]): + raise CallError("same_period", "基期和本期是同一期：写期次列里两个不同的值") + used = [*cols.values(), *([params["period"]] if "period" in params else [])] + if len(set(by)) != len(by) or set(by) & set(used) or len(set(used)) != len(used): + raise CallError("bad_by", "维度列、期次列、度量列要是不同的列") + return params, base, target, cols, by + + +def _period_rows(snap: Snap, period: str, wanted: Any, what: str) -> list[int]: + i = snap.index(period) + rows = [r for r, row in enumerate(snap.rows) if i < len(row) and same_value(wanted, row[i])] + if not rows: + values = list(dict.fromkeys(ph.key_text([row[i]]) for row in snap.rows if i < len(row))) + raise CallError("period_not_found", f"期次列「{period}」里没有「{wanted}」：{what}要写 {snap.ref} 里有的期次值，" + f"比如 {'、'.join(values[:6])}") + return rows + + +def _rows_of(raw: Mapping[str, Any], params: dict[str, Any], base: Snap, target: Snap) -> tuple[list[int], list[int]]: + if "period" in params: + return (_period_rows(base, params["period"], params["base"], "基期"), + _period_rows(target, params["period"], params["target"], "本期")) + return list(range(len(base.rows))), list(range(len(target.rows))) + + +def _collect(snap: Snap, rows: Sequence[int], by: Sequence[str], read: Any, labels: Sequence[str], + period_label: str | None) -> Side: + """一期的各段：按维度键，同一键只许一行（多行是 SQL 没按这些维度汇总）。read(row) 读出这一行的值（None 是空）。""" + side = Side(snap) + seen: set[tuple[Any, ...]] = set() + idx = [snap.index(c) for c in by] + for r in rows: + row = snap.rows[r] + key = tuple(hashable(row[i] if i < len(row) else None) for i in idx) + if key in seen: + keys = "、".join([*labels, *([period_label] if period_label else [])]) + raise CallError("duplicate_rows", f"{snap.ref} 里同一个{'、'.join(labels)}在同一期有不止一行：在 SQL 里按 " + f"{keys} 汇总，每段每期一行") + seen.add(key) + value = read(r) + side.rows.append(r) + if value is None: + side.nulls.append(key) + side.values[key] = value + return side + + +def _segments(base: Side, target: Side, zero: Any) -> list[Segment]: + """两期的各段，按先出现的顺序：缺的一边记 0，标「新出现 / 消失」；度量格为空的记 0，标「为空」。""" + out: list[Segment] = [] + for key in dict.fromkeys([*base.values, *target.values]): + flags = [] + if key not in base.values: + flags.append("新出现") + if key not in target.values: + flags.append("消失") + if key in base.nulls or key in target.nulls: + flags.append("为空") + b, t = base.values.get(key), target.values.get(key) + out.append(Segment(key, zero if b is None else b, zero if t is None else t, flags)) + return out + + +def _parent(raw: Mapping[str, Any], env: Env, mode: str, dims: int) -> tuple[dict[str, Any], list[Any], int]: + """下钻：上一层的结论和那一段。返回 (params 里的 parent, 那一段在上一层结果表里的那一行, 这一层的层数)。""" + from app.core.artifact_store import canonical_json, content_hash + + spec = raw.get("parent") + name = spec.get("of") if isinstance(spec, Mapping) else None + key = as_list(spec.get("key")) if isinstance(spec, Mapping) else [] + if not isinstance(name, str) or not name.strip() or not key: + raise CallError("bad_parent", PARENT_SHAPE) + name = name.strip() + found = env.parents(name) if env.parents is not None else None + if not isinstance(found, Mapping): + raise CallError("parent_missing", f"没有叫「{name}」的分析结论：下钻写这次运行里变化拆解的编号（如 A3）") + if found.get("op") != "decompose" or found.get("mode") != mode: + raise CallError("parent_not_decompose", f"{name} 不是{'按维度' if mode == 'additive' else '比率'}的变化拆解，" + "下钻要接在同一种拆解后面") + if found.get("status") != "ok" or not isinstance(found.get("table"), Mapping): + raise CallError("parent_refused", f"{name} 没有做成，不能在它下面下钻") + depth = int((found.get("params") or {}).get("depth") or 0) + 1 + limit = int(env.settings["max_drill_depth"]) + if depth > limit: + raise CallError("too_deep", f"已经下钻了 {depth - 1} 层，最多 {limit} 层：换一个维度从头拆") + table = found["table"] + fixed = _FIXED_ADDITIVE if mode == "additive" else _FIXED_RATIO + columns = list(table.get("columns") or []) + dim_count = len([c for c in columns if c not in fixed]) + if len(key) != dim_count: + raise CallError("bad_parent", f"{name} 按 {dim_count} 列拆，下钻的那一段要写 {dim_count} 个维度值") + hit = next((row for row in table.get("rows") or [] + if all(same_value(k, v) for k, v in zip(key, row[:dim_count]))), None) + if hit is None: + raise CallError("parent_key_missing", f"{name} 里没有「{ph.key_text(key)}」这一段：写 {name} 结果表里的段") + env.inputs.parent(name, content_hash(canonical_json(found))) + return {"of": name, "key": key}, dict(zip(columns, hit)), depth + + +_FIXED_ADDITIVE = ("base", "target", "delta", "pct", "share", "flag") +_FIXED_RATIO = ("n_base", "n_target", "d_base", "d_target", "w_base", "w_target", "r_base", "r_target", "mix", "rate", + "total", "flag") + + +def _headline(raw: Mapping[str, Any], env: Env) -> tuple[dict[str, str] | None, dict[str, tuple[Fraction, Any]]]: + """headline：基期、本期总数各一格 → ({base, target} 的写法, {base / target: (值, 原样)})。""" + spec = raw.get("headline") + if spec is None: + return None, {} + if isinstance(spec, (list, tuple)) and len(spec) == 2: + spec = {"base": spec[0], "target": spec[1]} + if not isinstance(spec, Mapping) or not spec.get("base") or not spec.get("target"): + raise CallError("bad_headline", HEADLINE_SHAPE) + out: dict[str, tuple[Fraction, Any]] = {} + refs: dict[str, str] = {} + for side, what in (("base", "基期总数"), ("target", "本期总数")): + snap, row, col, value = env.inputs.cell(spec[side], f"headline_{side}", what) + if value is None: + raise CallError("headline_empty", f"{what}那一格（{snap.ref}.r{row}.{col}）是空的：换一格有值的") + out[side] = (value, snap.cell(row, col)) + refs[side] = f"{snap.ref}.r{row}.{col}" + return refs, out + + +def _measure_guard(env: Env, snap: Snap, column: str, caveats: list[dict[str, str]], tail: str) -> None: + """additive 的度量列：目录标成（确认过的）比率、标识、状态、属性是调用错误；没标的、只是推断的照算，写附带说明。""" + kind, confirmed = env.facts.measure(snap, column) + if kind in _NOT_ADDITIVE and confirmed: + if kind == "ratio": + raise CallError("ratio_as_additive", f"「{column}」是比率（或平均值），不能按段相加：改用比率拆解，分子、分母各给一列") + raise CallError("not_a_measure", f"「{column}」在目录里是{_NOT_ADDITIVE[kind]}，不是能相加的量：度量列要选金额、数量这类列") + if kind is None: + caveats.append(ph.caveat("measure_unlabeled", ph.unlabeled_text(column, tail))) + elif kind in _NOT_ADDITIVE: + caveats.append(ph.caveat("measure_unlabeled", ph.unconfirmed_text(column, kind, tail))) + + +def _comparability(env: Env, params: dict[str, Any], base: Snap, target: Snap, measure: str | None + ) -> tuple[cmp.Outcome | None, str | None]: + """两份快照时先做可比性：(结论, 口径结构不同的原因)。""" + if "base_query" not in params: + return None, None + params["today"] = today_param(env) + outcome = cmp.compare_queries(base, target, measure, env.facts, env.book, env.today) # type: ignore[arg-type] + return outcome, outcome.structure + + +def _share_rank(segments: Sequence[Any], share: Any) -> list[Any]: + """按 share 从大到小（一样大的按先出现的）。""" + return sorted(segments, key=lambda s: -share(s)) + + +# -------------------------------------------------------------------------- +# additive +# -------------------------------------------------------------------------- + + +def additive(raw: Mapping[str, Any], env: Env) -> Any: + used = ["main_share", "concentration", "offset_ratio", "flat_pct", "table_top_k"] + params, bs, ts, cols, by = _sides(raw, env, [("measure", "度量列")]) + measure = cols["measure"] + label = label_for(env, raw.get("label"), ts, measure) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + dims = [column_label(env, ts, c) for c in by] + caveats: list[dict[str, str]] = [] + _measure_guard(env, ts, measure, caveats, "按可以按段相加处理") + refs, heads = _headline(raw, env) + if refs: + params["headline"] = refs + parent_row = None + if raw.get("parent") is not None: + params["parent"], parent_row, params["depth"] = _parent(raw, env, "additive", len(by)) + used.append("max_drill_depth") + summed = not heads and parent_row is None + criterion_args = {"summed": summed} + for snap in dict.fromkeys([bs, ts]): + if snap.truncated: + env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [*by, measure]) + return refuse(env, params, used, "truncated", + f"{snap.ref} 只取回了前 {len(snap.rows)} 行，各段之和不是完整的和", + criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args)) + rows_b, rows_t = _rows_of(raw, params, bs, ts) + period_label = column_label(env, ts, params["period"]) if "period" in params else None + side_b = _collect(bs, rows_b, by, lambda r: bs.number(r, measure, "度量列"), dims, period_label) + side_t = _collect(ts, rows_t, by, lambda r: ts.number(r, measure, "度量列"), dims, period_label) + if bs is ts: + env.inputs.use(ts, "data", [*by, params["period"], measure], side_b.rows + side_t.rows) + else: + env.inputs.use(bs, "base", [*by, measure], side_b.rows) + env.inputs.use(ts, "target", [*by, measure], side_t.rows) + outcome, structure = _comparability(env, params, bs, ts, measure) + if structure is not None: + return refuse(env, params, used, "incomparable_sides", f"{structure}，不是同一个量的两期，不拆", + criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats) + segs = _segments(side_b, side_t, Fraction(0)) + if len(segs) < 2: + return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(segs)} 段，至少要 2 段才拆得开", + criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats) + return _additive_result(env, params, used, segs, by, dims, label, heads, parent_row, outcome, caveats, summed, + side_b, side_t) + + +def _additive_result(env: Env, params: dict[str, Any], used: list[str], segs: list[Segment], by: list[str], + dims: list[str], label: str, heads: Mapping[str, tuple[Fraction, Any]], + parent_row: Mapping[str, Any] | None, outcome: cmp.Outcome | None, + caveats: list[dict[str, str]], summed: bool, side_b: Side, side_t: Side) -> Any: + s = env.settings + main_share, concentration = env.threshold("main_share"), env.threshold("concentration") + offset_ratio, flat_pct = env.threshold("offset_ratio"), env.threshold("flat_pct") + B = sum((x.base for x in segs), Fraction(0)) + T = sum((x.target for x in segs), Fraction(0)) + delta = T - B + gaps: dict[str, Fraction] = {} + for side, total in (("base", B), ("target", T)): + if side in heads and not closes(total, heads[side][0], heads[side][1]): + gaps[side] = heads[side][0] - total + if parent_row is not None: + expect = to_fraction(parent_row.get(side)) + if expect is None or not closes(total, expect, parent_row.get(side)): + gaps.setdefault(side, (expect or Fraction(0)) - total) + closed = not gaps + deltas = {x.key: x.target - x.base for x in segs} + shares = {k: d / delta for k, d in deltas.items()} if closed and delta != 0 else {} + gross_up = sum((d for d in deltas.values() if d > 0), Fraction(0)) + gross_down = sum((d for d in deltas.values() if d < 0), Fraction(0)) + opposite = abs(gross_down) if delta > 0 else gross_up if delta < 0 else gross_up + n = len(segs) + k_max = max(2, math.ceil(n / 5)) + flat = delta == 0 or (B != 0 and abs(delta) / abs(B) < flat_pct) + ranked = _share_rank(segs, lambda x: shares[x.key]) if shares else [] + top_k = top_k_share = None + if ranked: + running = Fraction(0) + for k, seg in enumerate(ranked, start=1): + running += shares[seg.key] + if running >= concentration: + top_k, top_k_share = k, running + break + if not closed: + code, subjects = "unclosed", [] + elif flat: + code, subjects = "flat", [] + elif shares[ranked[0].key] >= main_share: + code, subjects = "driver", [ranked[0]] + elif top_k is not None and 2 <= top_k <= k_max: + code, subjects = "concentrated", ranked[:top_k] + else: + code, subjects = "dispersed", [] + offset = opposite > 0 and opposite >= offset_ratio * abs(delta) + if offset: + caveats.append(ph.caveat("offset", ph.offset_text(delta, gross_up, gross_down))) + nulls = [x for x in segs if "为空" in x.flags] + if nulls: + caveats.append(ph.caveat("null_values", ph.null_segments_text([ph.key_text(x.key) for x in nulls], label))) + if summed: + caveats.append(ph.caveat("headline_basis", ph.HEADLINE_BASIS)) + if outcome is not None and outcome.verdict != "comparable": + caveats.append(ph.caveat("comparability", outcome.phrase)) + top = ranked[0] if ranked else max(segs, key=lambda x: abs(deltas[x.key])) + total_text = ph.change(delta, B) + top_n = int(s["table_top_k"]) + if code == "driver": + main = ph.decompose_driver(label, total_text, ph.key_text(top.key), ph.amount(deltas[top.key], signed=True), + shares[top.key]) + elif code == "concentrated": + main = ph.decompose_concentrated(label, total_text, ph.names([ph.key_text(x.key) for x in subjects], top_n), + top_k_share) + elif code == "dispersed": + main = ph.decompose_dispersed(label, total_text, ph.dims_text(dims), main_share) + elif code == "flat": + ups = [x for x in segs if deltas[x.key] > 0] + downs = [x for x in segs if deltas[x.key] < 0] + pair = None + if offset and ups and downs: + up = max(ups, key=lambda x: deltas[x.key]) + down = min(downs, key=lambda x: deltas[x.key]) + pair = (ph.key_text(up.key), ph.amount(deltas[up.key], signed=True), ph.key_text(down.key), + ph.amount(deltas[down.key], signed=True)) + main = ph.decompose_flat(label, delta / abs(B) if B != 0 else None, pair) + else: + gap = max(gaps.values(), key=abs) + main = ph.decompose_unclosed(ph.dims_text(dims), gap, ratio=False) + order = sorted(segs, key=lambda x: -abs(deltas[x.key])) + rows = [] + for x in order: + d = deltas[x.key] + rows.append([*x.key, canon(x.base), canon(x.target), canon(d), canon(d / x.base) if x.base != 0 else None, + canon(shares[x.key]) if shares else None, "、".join(x.flags) or None]) + table = {"columns": [*by, *_FIXED_ADDITIVE], + "labels": {**dict(zip(by, dims)), "base": "基期", "target": "本期", "delta": "变化", "pct": "变化率", + "share": "占变化的比重", "flag": "说明"}, + "rows": rows} + values: dict[str, Any] = { + "base_total": canon(B), "target_total": canon(T), "delta_total": canon(delta), + "pct_total": canon(delta / abs(B)) if B != 0 else None, "top_key": ph.key_text(top.key), + "top_share": canon(shares[top.key]) if shares else None, "top_k": str(top_k) if top_k is not None else None, + "top_k_share": canon(top_k_share) if top_k_share is not None else None, "gross_up": canon(gross_up), + "gross_down": canon(gross_down), "segments": str(n), + } + if gaps: + values["gap"] = canon(max(gaps.values(), key=abs)) + verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects], + "direction": sign_word(delta) if code in ("driver", "concentrated", "dispersed") else None} + phrase = ph.with_caveats(main, caveats, skip=("offset",) if code == "flat" else ()) + return build(env, params, used, table=table, values=values, verdict=verdict, + criterion=ph.criterion_additive(s, top_k_max=k_max, summed=summed), phrase=phrase, caveats=caveats) + + +# -------------------------------------------------------------------------- +# ratio +# -------------------------------------------------------------------------- + + +def ratio(raw: Mapping[str, Any], env: Env) -> Any: + used = ["main_share", "flat_pct", "min_denominator", "table_top_k"] + params, bs, ts, cols, by = _sides(raw, env, [("numerator", "分子列"), ("denominator", "分母列")]) + num, den = cols["numerator"], cols["denominator"] + num_label, den_label = column_label(env, ts, num), column_label(env, ts, den) + label = ph.pick_label(raw.get("label"), ph.say(num_label, "与", den_label, "之比")) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + dims = [column_label(env, ts, c) for c in by] + refs, heads = _headline(raw, env) + if refs: + params["headline"] = refs + parent_row = None + if raw.get("parent") is not None: + params["parent"], parent_row, params["depth"] = _parent(raw, env, "ratio", len(by)) + used.append("max_drill_depth") + criterion = ph.criterion_ratio(env.settings) + for snap in dict.fromkeys([bs, ts]): + if snap.truncated: + env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [*by, num, den]) + return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，各段合起来不是完整的数", + criterion=criterion) + rows_b, rows_t = _rows_of(raw, params, bs, ts) + period_label = column_label(env, ts, params["period"]) if "period" in params else None + + def reader(snap: Snap) -> Any: + return lambda r: (snap.number(r, num, "分子列"), snap.number(r, den, "分母列")) + + side_b = _collect(bs, rows_b, by, reader(bs), dims, period_label) + side_t = _collect(ts, rows_t, by, reader(ts), dims, period_label) + for side in (side_b, side_t): + side.nulls = [k for k, v in side.values.items() if v[0] is None or v[1] is None] + side.values = {k: (v[0] or Fraction(0), v[1] or Fraction(0)) for k, v in side.values.items()} + if bs is ts: + env.inputs.use(ts, "data", [*by, params["period"], num, den], side_b.rows + side_t.rows) + else: + env.inputs.use(bs, "base", [*by, num, den], side_b.rows) + env.inputs.use(ts, "target", [*by, num, den], side_t.rows) + outcome, structure = _comparability(env, params, bs, ts, den) + if structure is not None: + return refuse(env, params, used, "incomparable_sides", f"{structure}，不是同一个量的两期，不拆", criterion=criterion) + segs = _segments(side_b, side_t, (Fraction(0), Fraction(0))) + live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0] + if len(live) < 2: + return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(live)} 段，至少要 2 段才拆得开", + criterion=criterion) + for when, i in (("基期", 0), ("本期", 1)): + if sum(((x.base, x.target)[i][1] for x in live), Fraction(0)) == 0: + return refuse(env, params, used, "zero_base", ph.say(f"{when}的", den_label, "合计为 0，算不了比率"), + criterion=criterion) + for x in live: + for (n_, d_), when in ((x.base, "基期"), (x.target, "本期")): + if d_ == 0 and n_ != 0: + return refuse(env, params, used, "zero_base", + ph.say(ph.key_text(x.key), f"{when}的", den_label, "为 0 而", num_label, + "不为 0，比率拆不了"), criterion=criterion) + return _ratio_result(env, params, used, segs, by, dims, label, num_label, den_label, heads, parent_row, outcome, + criterion) + + +def ratio_effects(segs: Sequence[tuple[Fraction, Fraction, Fraction, Fraction]] + ) -> tuple[Fraction, Fraction, list[tuple[Fraction, ...]]]: + """比率拆解的计算（细案 §2.2，中点对称拆法）。segs 是各段 (分子_基期, 分母_基期, 分子_本期, 分母_本期)，两期分母都为 0 + 的段先剔掉。交回 (R, R', 各段 (w, w', r, r', mix, rate))：Σmix + Σrate = R' − R 逐位成立（Σ(w' − w) = 0）。一边分母为 0 + 的段：那一边权重 0、比率取另一边的（自身效应为 0，变化全记在结构上）。""" + db = sum((s[1] for s in segs), Fraction(0)) + dt = sum((s[3] for s in segs), Fraction(0)) + R = sum((s[0] for s in segs), Fraction(0)) / db + R2 = sum((s[2] for s in segs), Fraction(0)) / dt + mid = (R + R2) / 2 + out = [] + for nb, d_b, nt, d_t in segs: + w, w2 = d_b / db, d_t / dt + r = nb / d_b if d_b != 0 else None + r2 = nt / d_t if d_t != 0 else None + r = r2 if r is None else r + r2 = r if r2 is None else r2 + mix = (w2 - w) * ((r + r2) / 2 - mid) # type: ignore[operator] + rate = (r2 - r) * (w + w2) / 2 # type: ignore[operator] + out.append((w, w2, r, r2, mix, rate)) + return R, R2, out + + +def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[Segment], by: list[str], + dims: list[str], label: str, num_label: str, den_label: str, + heads: Mapping[str, tuple[Fraction, Any]], parent_row: Mapping[str, Any] | None, + outcome: cmp.Outcome | None, criterion: str) -> Any: + main_share, flat_pct = env.threshold("main_share"), env.threshold("flat_pct") + min_den = env.threshold("min_denominator") + live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0] + dropped = [x for x in segs if x not in live] + R, R2, effects = ratio_effects([(x.base[0], x.base[1], x.target[0], x.target[1]) for x in live]) + delta = R2 - R + by_key = {x.key: e for x, e in zip(live, effects)} + gaps: dict[str, Fraction] = {} + for side, total in (("base", R), ("target", R2)): + if side in heads and not closes(total, heads[side][0], heads[side][1]): + gaps[side] = heads[side][0] - total + if parent_row is not None: + sums = {"n_base": sum((x.base[0] for x in live), Fraction(0)), "d_base": sum((x.base[1] for x in live), Fraction(0)), + "n_target": sum((x.target[0] for x in live), Fraction(0)), + "d_target": sum((x.target[1] for x in live), Fraction(0))} + for col, total in sums.items(): + expect = to_fraction(parent_row.get(col)) + if expect is None or not closes(total, expect, parent_row.get(col)): + gaps.setdefault("base" if col.endswith("base") else "target", (expect or Fraction(0)) - total) + closed = not gaps + mix_total = sum((e[4] for e in effects), Fraction(0)) + rate_total = sum((e[5] for e in effects), Fraction(0)) + flat = delta == 0 or (R != 0 and abs(delta) / abs(R) < flat_pct) + mix_share = mix_total / delta if delta != 0 else None + rate_share = rate_total / delta if delta != 0 else None + sign = 1 if delta >= 0 else -1 + if not closed: + code, subjects = "unclosed", [] + elif flat: + code, subjects = "flat", [] + elif mix_share >= main_share and rate_share < main_share: # type: ignore[operator] + code = "mix_driven" + subjects = [max(live, key=lambda x: sign * by_key[x.key][4])] + elif rate_share >= main_share and mix_share < main_share: # type: ignore[operator] + code = "rate_driven" + subjects = [max(live, key=lambda x: sign * by_key[x.key][5])] + else: + code, subjects = "both", [] + proportion = all(0 <= v[0] <= v[1] for x in live for v in (x.base, x.target)) + small = [x for x in live if x.base[1] < min_den or x.target[1] < min_den] + caveats: list[dict[str, str]] = [] + eligible = [x for x in live if x not in small] + if delta != 0 and eligible: + weight = sum(((by_key[x.key][0] + by_key[x.key][1]) / 2 for x in eligible), Fraction(0)) + against = sum(((by_key[x.key][0] + by_key[x.key][1]) / 2 for x in eligible + if (by_key[x.key][3] - by_key[x.key][2]) * sign < 0), Fraction(0)) + if weight > 0 and against / weight >= SIMPSON_SHARE: + star = max(live, key=lambda x: sign * by_key[x.key][4]) + caveats.append(ph.caveat("simpson", ph.simpson_text(label, delta > 0, ph.key_text(star.key), + by_key[star.key][0], by_key[star.key][1]))) + if small: + caveats.append(ph.caveat("small_denominator", ph.small_denominator_text( + [ph.key_text(x.key) for x in small], den_label, int(env.settings["min_denominator"])))) + nulls = [x for x in segs if "为空" in x.flags] + if nulls: + caveats.append(ph.caveat("null_values", ph.null_segments_text([ph.key_text(x.key) for x in nulls], + ph.say(num_label, "或", den_label)))) + if outcome is not None and outcome.verdict != "comparable": + caveats.append(ph.caveat("comparability", outcome.phrase)) + total_text = ph.ratio_change(R, R2, proportion=proportion) + if code == "mix_driven": + star = subjects[0] + main = ph.ratio_mix(label, total_text, ph.key_text(star.key), by_key[star.key][0], by_key[star.key][1]) + elif code == "rate_driven": + main = ph.ratio_rate(label, total_text, ph.key_text(subjects[0].key)) + elif code == "both": + main = ph.ratio_both(label, total_text, mix_share, rate_share) + elif code == "flat": + main = ph.decompose_flat(label, delta / abs(R) if R != 0 else None, None) + else: + main = ph.decompose_unclosed(ph.dims_text(dims), max(gaps.values(), key=abs), ratio=True) + rows = [] + for x in sorted(live, key=lambda x: -abs(by_key[x.key][4] + by_key[x.key][5])): + w, w2, r, r2, mix, rate = by_key[x.key] + flags = list(x.flags) + if x.base[1] == 0: + flags.append("基期分母为 0") + if x.target[1] == 0: + flags.append("本期分母为 0") + if x in small: + flags.append("分母小") + rows.append([*x.key, canon(x.base[0]), canon(x.target[0]), canon(x.base[1]), canon(x.target[1]), canon(w), + canon(w2), canon(r), canon(r2), canon(mix), canon(rate), canon(mix + rate), + "、".join(dict.fromkeys(flags)) or None]) + for x in dropped: + rows.append([*x.key, canon(x.base[0]), canon(x.target[0]), "0", "0", None, None, None, None, None, None, None, + "两期分母都为 0，不计入"]) + table = {"columns": [*by, *_FIXED_RATIO], + "labels": {**dict(zip(by, dims)), "n_base": ph.say("基期", num_label), "n_target": ph.say("本期", num_label), + "d_base": ph.say("基期", den_label), "d_target": ph.say("本期", den_label), + "w_base": "基期权重", "w_target": "本期权重", "r_base": "基期比率", "r_target": "本期比率", + "mix": "结构效应", "rate": "自身效应", "total": "合计影响", "flag": "说明"}, + "rows": rows} + values: dict[str, Any] = { + "r_base_total": canon(R), "r_target_total": canon(R2), "delta": canon(delta), "mix_total": canon(mix_total), + "rate_total": canon(rate_total), "mix_share": canon(mix_share), "rate_share": canon(rate_share), + "segments": str(len(live)), + } + if gaps: + values["gap"] = canon(max(gaps.values(), key=abs)) + verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects], + "direction": sign_word(delta) if code in ("mix_driven", "rate_driven", "both") else None} + return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, + phrase=ph.with_caveats(main, caveats), caveats=caveats) + + +# -------------------------------------------------------------------------- +# factor +# -------------------------------------------------------------------------- + + +def _half_ulp_ratio(raw: Any, value: Fraction) -> Fraction: + from app.analysis.common import written_places + + return Fraction(1, 2) / Fraction(10) ** written_places(raw) / abs(value) if value != 0 else Fraction(0) + + +def lmdi(base: Sequence[float], target: Sequence[float], m_base: float, m_target: float) -> list[float]: + """LMDI-I：contrib_k = L(M', M) · ln(F_k' / F_k)，L(a, b) = (a − b) / (ln a − ln b)，L(a, a) = a。因子按书写精度取过整、 + 乘积和乘积列差一点时，各因子按同一比例缩放到乘积恰好等于乘积列（每个因子的对数比各补上 (1/K)·差的对数），Σcontrib 在 + 浮点误差内等于 M' − M。""" + k = len(base) + fix = (math.log(m_target / math.prod(target)) - math.log(m_base / math.prod(base))) / k + weight = m_base if m_base == m_target else (m_target - m_base) / (math.log(m_target) - math.log(m_base)) + return [weight * (math.log(t / b) + fix) for b, t in zip(base, target)] + + +def factor(raw: Mapping[str, Any], env: Env) -> Any: + used = ["main_share", "flat_pct"] + params, bs, ts, cols, _ = _sides(raw, env, [("product", "乘积列")], by_needed=False) + product = cols["product"] + wanted = as_list(raw.get("factors")) + if not 2 <= len(wanted) <= 4: + raise CallError("bad_factors", "因子列要写 2 到 4 列") + factors = [ts.column(c, "因子列") for c in wanted] + if bs is not ts: + for c in factors: + bs.column(c, "因子列") + if len(set(factors)) != len(factors) or product in factors: + raise CallError("bad_factors", "因子列不能重复，也不能是乘积列本身") + params["factors"] = factors + label = label_for(env, raw.get("label"), ts, product) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + names = [column_label(env, ts, c) for c in factors] + criterion = ph.criterion_factor(env.settings) + for snap in dict.fromkeys([bs, ts]): + if snap.truncated: + env.inputs.use(snap, "data" if bs is ts else ("base" if snap is bs else "target"), [product, *factors]) + return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，结果不完整", + criterion=criterion) + if "period" in params: + rows_b = _period_rows(bs, params["period"], params["base"], "基期") + rows_t = _period_rows(ts, params["period"], params["target"], "本期") + else: + rows_b, rows_t = list(range(len(bs.rows))), list(range(len(ts.rows))) + for rows, snap, when in ((rows_b, bs, "基期"), (rows_t, ts, "本期")): + if len(rows) != 1: + raise CallError("duplicate_rows", f"{snap.ref} 里{when}有 {len(rows)} 行：乘积拆解每期要正好一行（在 SQL 里汇总成一行）") + cells: dict[str, list[tuple[Fraction, Any]]] = {} + for when, snap, r in (("base", bs, rows_b[0]), ("target", ts, rows_t[0])): + row = [] + for col, what in ((product, "乘积列"), *((c, "因子列") for c in factors)): + value = snap.number(r, col, what) + if value is None: + raise CallError("empty_cell", f"{snap.ref} 第 {r} 行的「{col}」是空的：乘积、因子都要有值") + row.append((value, snap.cell(r, col))) + cells[when] = row + if bs is ts: + env.inputs.use(ts, "data", [params["period"], product, *factors], rows_b + rows_t) + else: + env.inputs.use(bs, "base", [product, *factors], rows_b) + env.inputs.use(ts, "target", [product, *factors], rows_t) + outcome, structure = _comparability(env, params, bs, ts, product) + if structure is not None: + return refuse(env, params, used, "incomparable_sides", f"{structure}，不是同一个量的两期，不拆", criterion=criterion) + if any(v <= 0 for row in cells.values() for v, _ in row): + return refuse(env, params, used, "non_positive", "乘积或因子里有不大于 0 的值，对数拆解要求全为正", criterion=criterion) + for when, row in cells.items(): + m = row[0][0] + prod = math.prod((v for v, _ in row[1:]), start=Fraction(1)) + rel = abs(m - prod) / abs(m) + allowed = max(_FACTOR_REL, sum((_half_ulp_ratio(raw_v, v) for v, raw_v in row[1:]), Fraction(0))) + if rel > allowed: + raise CallError("not_product", ph.say("这几列相乘不等于", column_label(env, ts, product), + f"（{'基期' if when == 'base' else '本期'}差 {ph.pct(rel, signed=False)}），" + "不是乘积关系：换成乘起来正好等于它的几列")) + M, M2 = cells["base"][0][0], cells["target"][0][0] + fb = [float(v) for v, _ in cells["base"][1:]] + ft = [float(v) for v, _ in cells["target"][1:]] + contrib = lmdi(fb, ft, float(M), float(M2)) + delta = M2 - M + flat = delta == 0 or abs(delta) / abs(M) < env.threshold("flat_pct") + shares = [c / float(delta) for c in contrib] if delta != 0 else [None] * len(contrib) + main_share = float(env.threshold("main_share")) + order = sorted(range(len(factors)), key=lambda i: -abs(contrib[i])) + caveats: list[dict[str, str]] = [] + if outcome is not None and outcome.verdict != "comparable": + caveats.append(ph.caveat("comparability", outcome.phrase)) + total_text = ph.change(delta, M) + top = max(range(len(factors)), key=lambda i: shares[i] if shares[i] is not None else -math.inf) + if flat: + code, subjects = "flat", [] + main = ph.decompose_flat(label, delta / abs(M), None) + elif shares[top] >= main_share: # type: ignore[operator] + code, subjects = "driver", [[names[top]]] + main = ph.factor_driver(label, total_text, names[top], contrib[top], shares[top]) + else: + code, subjects = "both", [] + main = ph.factor_both(label, total_text, [(names[i], shares[i]) for i in order]) + rows = [] + for i in order: + b, t = cells["base"][i + 1][0], cells["target"][i + 1][0] + rows.append([names[i], canon(b), canon(t), canon((t - b) / b), canon(contrib[i]), + canon(shares[i]) if shares[i] is not None else None]) + table = {"columns": ["factor", "base", "target", "pct", "contrib", "share"], + "labels": {"factor": "因子", "base": "基期", "target": "本期", "pct": "变化率", "contrib": "贡献", + "share": "占变化的比重"}, + "rows": rows} + values = {"base_total": canon(M), "target_total": canon(M2), "delta_total": canon(delta), + "pct_total": canon(delta / abs(M)), "top_key": names[top], + "top_share": canon(shares[top]) if shares[top] is not None else None} + verdict = {"code": code, "subjects": subjects, "direction": sign_word(delta) if code != "flat" else None} + return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, + phrase=ph.with_caveats(main, caveats), caveats=caveats) + + +__all__ = ["HEADLINE_SHAPE", "PARENT_SHAPE", "SAME_QUERY", "SIMPSON_SHARE", "additive", "factor", "lmdi", "ratio", "ratio_effects"] diff --git a/backend/app/analysis/differ.py b/backend/app/analysis/differ.py new file mode 100644 index 00000000..a6d5e7ef --- /dev/null +++ b/backend/app/analysis/differ.py @@ -0,0 +1,372 @@ +"""差异检验（细案 01 §5）：这两组（或两期）真的不一样，还是样本波动。 + +**参数**：query（一份快照）；按组比时 by（分组列，一列）；按期比同一组时 period、base、target（各一行，SQL 里先筛好那一组）。 +比哪几对：pairs（[[A, B], …]，A、B 是分组列里的值）、reference（对照组：其余各组各和它比）；都不写时两组就比这两组，三组 +以上每组和「其余各组合计」比（和含它自己的整体比不独立）。可选 label。 + +- rate：numerator（成功数，如转化单数）、denominator（试验数，如访客数），每组一行，都要是不小于 0 的整数、成功数不超过 + 试验数。每组 Wilson 区间，两组之差 Newcombe 区间；最小期望频数不小于 min_expected 用合并比例的 z 检验，否则用 Fisher + 精确检验。 +- mean：每组一行，mean、sd、n 三列（均值、标准差、样本数），或者 sum、sumsq、n 三列（合计、平方和、样本数：SQLite 没有标准差 + 函数，SUM(x)、SUM(x*x)、COUNT(*) 都写得出）。Welch t 检验，自由度按 Welch–Satterthwaite。 +- rows：value（逐行的值）+ 分组列（或期次列），快照不超过 1000 行、没截断；算子自己算均值、方差，另做敏感性：去掉离均值 + 最远的那一个值后重算，判定变了加 outlier_sensitive。 + +比了几对时按 Holm 校正，表里同时给原始 p 和校正后的 p，附 multiple_comparisons。校正后 p < differ_alpha → differs（direction +按 A − B 的正负）；否则 no_evidence。用了 Fisher（比率）或有一组样本数小于 min_n（均值）→ small_sample。判定语不出现「p 值」 +「显著性」：differs 说「差异站得住」，no_evidence 说「差异可能只是样本波动」，p 值、区间在表和证据面板里。 + +subjects 是判定说的那一对 [[A], [B]]（只比一对时就是那一对，比了几对时是校正后最站得住的那一对；几对都不站得住时为空）， +direction 是 A 相对 B。 +""" +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from fractions import Fraction +from typing import Any + +from app.analysis import phrases as ph +from app.analysis import stats +from app.analysis.common import Env, build, column_label, label_for, need, refuse, sign_word +from app.analysis.contract import CallError +from app.analysis.inputs import Snap, as_list, hashable, same_value +from app.analysis.numbers import canon, to_fraction + +TESTS = {"z": "z 检验", "fisher": "Fisher 精确检验", "welch": "Welch t 检验"} + + +@dataclass(eq=False) +class Group: + key: Any + name: str + rows: list[int] = field(default_factory=list) + x: int = 0 # rate：成功数 + n: int = 0 # rate：试验数；mean / rows：样本数 + mean: Fraction | None = None + var: Fraction | None = None + values: list[Fraction] = field(default_factory=list) + + +@dataclass +class Pair: + a: Group + b: Group + test: str | None = None + p: float | None = None + p_adj: float | None = None + diff: Fraction | None = None + ci: tuple[float, float] | None = None + a_value: Fraction | None = None + b_value: Fraction | None = None + + +def _int(value: Fraction | None, snap: Snap, r: int, column: str) -> int: + if value is None or value.denominator != 1 or value < 0: + raise CallError("not_count", f"{snap.ref} 第 {r} 行的「{column}」不是不小于 0 的整数：比率比较的分子、分母要是计数") + return int(value) + + +def _pool(name: str, groups: Sequence[Group], mode: str) -> Group: + """「其余各组合计」：比率把成功数、试验数加起来；均值按各组的均值、方差、样本数合并；逐行的值并在一起。""" + out = Group(("__rest__",), name, rows=[r for g in groups for r in g.rows]) + if mode == "rate": + out.x, out.n = sum(g.x for g in groups), sum(g.n for g in groups) + return out + if mode == "rows": + out.values = [v for g in groups for v in g.values] + return _moments(out) + out.n = sum(g.n for g in groups) + if out.n == 0: + return out + out.mean = sum((g.mean * g.n for g in groups if g.mean is not None), Fraction(0)) / out.n + if out.n > 1: + within = sum(((g.n - 1) * g.var for g in groups if g.var is not None and g.n > 1), Fraction(0)) + between = sum((g.n * (g.mean - out.mean) ** 2 for g in groups if g.mean is not None), Fraction(0)) + out.var = (within + between) / (out.n - 1) + return out + + +def _moments(group: Group) -> Group: + group.n = len(group.values) + if group.n: + group.mean = sum(group.values, Fraction(0)) / group.n + if group.n > 1: + group.var = sum(((v - group.mean) ** 2 for v in group.values), Fraction(0)) / (group.n - 1) # type: ignore[operator] + return group + + +def _test(pair: Pair, mode: str, min_expected: Fraction) -> None: + a, b = pair.a, pair.b + if mode == "rate": + if a.n == 0 or b.n == 0: + return + pair.a_value, pair.b_value = Fraction(a.x, a.n), Fraction(b.x, b.n) + pair.diff = pair.a_value - pair.b_value + pair.ci = stats.newcombe(a.x, a.n, b.x, b.n) + if stats.min_expected(a.x, a.n, b.x, b.n) >= min_expected: + pair.test = "z" + pair.p = stats.pooled_z(a.x, a.n, b.x, b.n)[1] # type: ignore[index] + else: + pair.test = "fisher" + pair.p = stats.fisher_exact(a.x, a.n - a.x, b.x, b.n - b.x) + return + if a.mean is None or b.mean is None: + return + pair.a_value, pair.b_value = a.mean, b.mean + pair.diff = a.mean - b.mean + if a.var is None or b.var is None: + return + test = stats.welch(float(a.mean), float(a.var), a.n, float(b.mean), float(b.var), b.n) + if test is None: + return + pair.test, pair.p = "welch", test.p + pair.ci = stats.welch_interval(float(a.mean), float(a.var), a.n, float(b.mean), float(b.var), b.n) + + +def _verdict(pairs: Sequence[Pair], alpha: Fraction) -> list[Pair]: + """Holm 校正后站得住的那几对（按校正后的 p 从小到大，一样的按先后）。""" + adjusted = stats.holm([p.p for p in pairs]) + for pair, adj in zip(pairs, adjusted): + pair.p_adj = adj + hits = [p for p in pairs if p.p_adj is not None and to_fraction(p.p_adj) < alpha] # type: ignore[operator] + return sorted(hits, key=lambda p: p.p_adj) # type: ignore[arg-type, return-value] + + +def differ(raw: Mapping[str, Any], env: Env) -> Any: + mode = env.mode + used = ["differ_alpha", "min_expected"] if mode == "rate" else ["differ_alpha", "min_n"] + snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号") + params: dict[str, Any] = {"query": snap.ref} + by_period = raw.get("period") is not None + if by_period: + group_col = snap.column(raw.get("period"), "期次列") + params.update(period=group_col, base=need(raw, "base", "基期（期次列里的值）"), + target=need(raw, "target", "本期（期次列里的值）")) + else: + wanted = as_list(raw.get("by")) + if len(wanted) != 1: + raise CallError("bad_by", "差异检验按一列分组：分组列写一列；比同一组的两期就写期次列、基期、本期") + group_col = snap.column(wanted[0], "分组列") + params["by"] = [group_col] + if mode == "rate": + cols = {"numerator": snap.column(need(raw, "numerator", "成功数列"), "成功数列"), + "denominator": snap.column(need(raw, "denominator", "试验数列"), "试验数列")} + elif mode == "mean": + if raw.get("mean") is not None: + cols = {"mean": snap.column(raw.get("mean"), "均值列"), "sd": snap.column(need(raw, "sd", "标准差列"), "标准差列"), + "n": snap.column(need(raw, "n", "样本数列"), "样本数列")} + else: + cols = {"sum": snap.column(need(raw, "sum", "合计列（或均值列）"), "合计列"), + "sumsq": snap.column(need(raw, "sumsq", "平方和列"), "平方和列"), + "n": snap.column(need(raw, "n", "样本数列"), "样本数列")} + else: + cols = {"value": snap.column(need(raw, "value", "值列"), "值列")} + params.update(cols) + measure_col = cols.get("numerator") or cols.get("mean") or cols.get("sum") or cols.get("value") + if mode == "rate": + label = ph.pick_label(raw.get("label"), ph.say(column_label(env, snap, cols["numerator"]), "占", + column_label(env, snap, cols["denominator"]), "的比率")) + else: + label = label_for(env, raw.get("label"), snap, measure_col) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + dim_label = column_label(env, snap, group_col) + criterion = ph.criterion_differ(env.settings, mode) # type: ignore[arg-type] + if snap.truncated: + env.inputs.use(snap, "data", [group_col, *cols.values()]) + return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，各组不完整", + criterion=criterion) + groups = _groups(snap, group_col, cols, mode, by_period, params) # type: ignore[arg-type] + pairs = _pairs(raw, params, groups, mode, dim_label, by_period, group_col) # type: ignore[arg-type] + env.inputs.use(snap, "data", [group_col, *cols.values()], + sorted({r for p in pairs for g in (p.a, p.b) for r in g.rows})) + return _decide(env, params, used, pairs, mode, label, criterion) # type: ignore[arg-type] + + +def _groups(snap: Snap, group_col: str, cols: Mapping[str, str], mode: str, by_period: bool, + params: Mapping[str, Any]) -> dict[Any, Group]: + gi = snap.index(group_col) + groups: dict[Any, Group] = {} + wanted = [params["target"], params["base"]] if by_period else None + for r, row in enumerate(snap.rows): + value = row[gi] if gi < len(row) else None + if wanted is not None: + if not any(same_value(w, value) for w in wanted): + continue + key = hashable(value) + group = groups.setdefault(key, Group(key, ph.key_text([value]))) + if mode != "rows" and group.rows: + raise CallError("duplicate_rows", f"{snap.ref} 里「{ph.key_text([value])}」有不止一行：在 SQL 里按 {group_col} " + "汇总，每组一行") + group.rows.append(r) + if mode == "rate": + group.x = _int(snap.number(r, cols["numerator"], "成功数列"), snap, r, cols["numerator"]) + group.n = _int(snap.number(r, cols["denominator"], "试验数列"), snap, r, cols["denominator"]) + if group.x > group.n: + raise CallError("not_count", f"{snap.ref} 第 {r} 行的成功数大于试验数：分子、分母要是同一批里的「成功几个、" + "一共几个」") + elif mode == "mean": + n = _int(snap.number(r, cols["n"], "样本数列"), snap, r, cols["n"]) + group.n = n + if "mean" in cols: + group.mean = snap.number(r, cols["mean"], "均值列") + sd = snap.number(r, cols["sd"], "标准差列") + group.var = sd * sd if sd is not None and n > 1 else None + else: + total = snap.number(r, cols["sum"], "合计列") + sq = snap.number(r, cols["sumsq"], "平方和列") + if total is not None and n > 0: + group.mean = total / n + if sq is not None and n > 1: + group.var = max(Fraction(0), (sq - total * total / n) / (n - 1)) + else: + v = snap.number(r, cols["value"], "值列") + if v is not None: + group.values.append(v) + if mode == "rows": + for g in groups.values(): + _moments(g) + if by_period: + for w, what in ((params["target"], "本期"), (params["base"], "基期")): + if not any(same_value(w, g.key) for g in groups.values()): + raise CallError("period_not_found", f"期次列「{group_col}」里没有「{w}」：{what}要写 {snap.ref} 里有的期次值") + return groups + + +def _find(groups: Mapping[Any, Group], value: Any, column: str) -> Group: + found = next((g for g in groups.values() if same_value(value, g.key)), None) + if found is None: + shown = "、".join(g.name for g in list(groups.values())[:8]) + raise CallError("unknown_group", f"「{value}」不是「{column}」里的值：要比的组写 {shown} 里的") + return found + + +def _pairs(raw: Mapping[str, Any], params: dict[str, Any], groups: dict[Any, Group], mode: str, dim_label: str, + by_period: bool, group_col: str) -> list[Pair]: + if by_period: + target = _find(groups, params["target"], group_col) + base = _find(groups, params["base"], group_col) + return [Pair(target, base)] + if raw.get("pairs") is not None: + spec = raw.get("pairs") + if not isinstance(spec, list) or not spec or not all(isinstance(p, (list, tuple)) and len(p) == 2 for p in spec): + raise CallError("bad_pairs", "要比的几对写成 [[A, B], …]，A、B 是分组列里的值") + out = [] + for a, b in spec: + ga, gb = _find(groups, a, group_col), _find(groups, b, group_col) + if ga is gb: + raise CallError("bad_pairs", f"「{ga.name}」不能和自己比") + out.append(Pair(ga, gb)) + params["pairs"] = [[p.a.key, p.b.key] for p in out] + return out + if len(groups) < 2: + raise CallError("too_few_groups", f"「{group_col}」里只有 {len(groups)} 组，至少要两组才比得了") + if raw.get("reference") is not None: + ref = _find(groups, raw.get("reference"), group_col) + params["reference"] = ref.key + return [Pair(g, ref) for g in groups.values() if g is not ref] + items = list(groups.values()) + if len(items) == 2: + return [Pair(items[0], items[1])] + rest_name = ph.say("其余各", dim_label) + return [Pair(g, _pool(rest_name, [o for o in items if o is not g], mode)) for g in items] + + +def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair], mode: str, label: str, + criterion: str) -> Any: + alpha = env.threshold("differ_alpha") + min_expected = env.threshold("min_expected") if mode == "rate" else Fraction(0) + for pair in pairs: + _test(pair, mode, min_expected) + hits = _verdict(pairs, alpha) + caveats: list[dict[str, str]] = [] + if mode == "rows": + flipped = _sensitivity(pairs, mode, alpha, min_expected, bool(hits)) + if flipped is not None: + caveats.append(ph.caveat("outlier_sensitive", ph.outlier_text(*flipped))) + hits = _verdict(pairs, alpha) + if mode == "rate" and any(p.test == "fisher" for p in pairs): + caveats.append(ph.caveat("small_sample", ph.small_sample_rate_text(int(env.settings["min_expected"])))) + if mode != "rate": + min_n = int(env.settings["min_n"]) + small = sorted((g for p in pairs for g in (p.a, p.b) if g.n < min_n), key=lambda g: g.n) + if small: + caveats.append(ph.caveat("small_sample", ph.small_sample_mean_text(small[0].name, small[0].n, min_n))) + tested = [p for p in pairs if p.p is not None] + if len(tested) > 1: + caveats.append(ph.caveat("multiple_comparisons", ph.multiple_text(len(tested)))) + fmt = (lambda v: ph.pct(v, signed=False)) if mode == "rate" else (lambda v: ph.amount(v)) + if hits: + code = "differs" + head = hits[0] + main = ph.differ_phrase([ph.differ_pair(p.a.name, p.b.name, label, fmt(p.a_value), fmt(p.b_value), + p.diff > 0) for p in hits], # type: ignore[operator] + len(pairs)) + subjects = [[head.a.name], [head.b.name]] + direction = sign_word(head.diff) # type: ignore[arg-type] + else: + code = "no_evidence" + head = pairs[0] if len(pairs) == 1 else None + sizes = (ph.amount(head.a.n), ph.amount(head.b.n)) if head is not None else None + main = ph.no_evidence_phrase(head.a.name if head else None, head.b.name if head else None, label, sizes, + len(pairs)) + subjects = [[head.a.name], [head.b.name]] if head is not None else [] + direction = None + value_name = "比率" if mode == "rate" else "均值" + rows = [] + for p in pairs: + flags = [] + if p.p is None: + flags.append("样本不够，算不了") + if p.test == "fisher": + flags.append("样本少，用了精确检验") + rows.append([p.a.name, p.b.name, canon(p.a_value), canon(p.b_value), canon(p.diff), + canon(p.ci[0]) if p.ci else None, canon(p.ci[1]) if p.ci else None, str(p.a.n), str(p.b.n), + TESTS.get(p.test or "") or None, canon(p.p), canon(p.p_adj), "、".join(flags) or None]) + table = {"columns": ["a", "b", "a_value", "b_value", "diff", "ci_lo", "ci_hi", "a_n", "b_n", "test", "p", "p_adj", + "flag"], + "labels": {"a": "组", "b": "对照组", "a_value": f"组的{value_name}", "b_value": f"对照组的{value_name}", + "diff": "差", "ci_lo": "差的区间下限", "ci_hi": "差的区间上限", "a_n": "组的样本量", + "b_n": "对照组的样本量", "test": "检验", "p": "p 值", "p_adj": "校正后的 p 值", "flag": "说明"}, + "rows": rows} + shown = head if head is not None else pairs[0] + values = {"a": canon(shown.a_value), "b": canon(shown.b_value), "diff": canon(shown.diff), + "ci_lo": canon(shown.ci[0]) if shown.ci else None, "ci_hi": canon(shown.ci[1]) if shown.ci else None, + "p": canon(shown.p), "p_adj": canon(shown.p_adj), "n_a": str(shown.a.n), "n_b": str(shown.b.n), + "test": shown.test, "pairs": str(len(pairs)), "differs": str(len(hits))} + verdict = {"code": code, "subjects": subjects, "direction": direction} + phrase = ph.with_caveats(main, caveats, skip=("multiple_comparisons",) if len(pairs) > 1 else ()) + return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, phrase=phrase, + caveats=caveats) + + +def _sensitivity(pairs: list[Pair], mode: str, alpha: Fraction, min_expected: Fraction, + differs: bool) -> tuple[bool, bool] | None: + """rows 模式的敏感性：去掉比较的各组里离均值最远的那一个值重算，判定变了交回 (去掉的是不是偏大的那一笔, 现在站不站得住)。""" + everything = [(g, i, v) for g in dict.fromkeys(g for p in pairs for g in (p.a, p.b)) for i, v in enumerate(g.values)] + if len(everything) < 3: + return None + mean = sum((v for _, _, v in everything), Fraction(0)) / len(everything) + group, index, value = max(everything, key=lambda t: abs(t[2] - mean)) + trimmed = [] + for p in pairs: + def cut(g: Group) -> Group: + if g is not group and not (g.key == ("__rest__",) and any(v is value for v in g.values)): + return g + values = list(g.values) + for k, v in enumerate(values): + if v is value: + del values[k] + break + return _moments(Group(g.key, g.name, rows=g.rows, values=values)) + trimmed.append(Pair(cut(p.a), cut(p.b))) + for p in trimmed: + _test(p, mode, min_expected) + now = bool(_verdict(trimmed, alpha)) + _verdict(pairs, alpha) + return (value > mean, now) if now != differs else None + + +__all__ = ["TESTS", "differ"] diff --git a/backend/app/analysis/inputs.py b/backend/app/analysis/inputs.py index 25d82152..88229117 100644 --- a/backend/app/analysis/inputs.py +++ b/backend/app/analysis/inputs.py @@ -198,6 +198,8 @@ class Inputs: out["artifact"] = snap.artifact if snap.dialect: out["dialect"] = snap.dialect + if snap.masked: + out["masked"] = sorted(snap.masked) # 只记列名（复算按它认遮罩列，快照工件里不一定都有） out.update({k: v for k, v in entry.items() if k != "ref"}) if out not in self.records: self.records.append(out) # type: ignore[arg-type] diff --git a/backend/app/analysis/series.py b/backend/app/analysis/series.py new file mode 100644 index 00000000..8bf7267a --- /dev/null +++ b/backend/app/analysis/series.py @@ -0,0 +1,304 @@ +"""放回历史看（细案 01 §3、§4）：波动基线（这个变化大不大、算不算异常）和趋势（是持续在变，还是一次波动）。 + +**参数**：query（一份快照：一条序列；要看某一段就在 SQL 里筛好）、period（期次列）、measure（度量列）；可选 grain（day / +week / month / quarter，缺省按期次值推断，见 analysis/periods）、label。波动基线另有 target（本期，缺省最后一期）、basis +（change 缺省：和比较期比的变化率；level：值本身，给存量、比率用）、lag（和往前第几期比，缺省日数据 7、其余 1；同比写 52 / +12 / 4，历史要相应够长）。 + +**前提**（不满足就算不了，原因照实写）：期次认得出（period_unreadable）、本期完整（同可比性检查的整期：没过完、数据没到、 +SQL 截掉一部分 → partial_target；表的日期范围查不到时只按运行当天判，附一句核对不了）、本期和比较期之间不缺期（target_gap）、 +比较期不为 0（zero_base）、历史够长（history_short）；趋势要最后几期连着（中间缺期 series_gap）。度量格为空的那一期跳过。 + +**波动基线**：历史的中位数 m、中位绝对偏差 MAD，σ̂ = 1.4826·MAD，稳健 z = (x − m)/σ̂；常规波动范围 m ± k_warn·σ̂；全程 +精确的有理数。σ̂ = 0（历史每期都一样）时本期一样判常规、不一样判明显超出。本期的值（不论 basis）是历史各期里最高 / 最低的 +加 record_high / record_low：「历史新高」说的是值本身。为什么不用均值和标准差：促销周这类极端值会把它们拉宽。 + +**趋势**：最后 n 期（期次连着的那一段，trend_min_points ≤ n ≤ max_history + 1）的 Theil–Sen 斜率、Mann–Kendall 双侧 p， +相对斜率 = 斜率 / |序列中位数|。p < trend_alpha 时按斜率的正负判上升、下降。末尾连续同向的期数从最后一期往前数（变化为 0 +断开）。已知局限：Mann–Kendall 假设各期独立，业务序列常有自相关，p 会偏小；所以缺省 n 至少 6、α 取 0.05，判定语只说 +「呈上升趋势」，不说「显著上升」。日数据附一句星期效应。 +""" +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from fractions import Fraction +from typing import Any + +from app.analysis import comparable as cmp +from app.analysis import periods as pd +from app.analysis import phrases as ph +from app.analysis import stats +from app.analysis.common import Env, build, column_label, label_for, need, refuse, sign_word, today_param +from app.analysis.contract import CallError +from app.analysis.inputs import Snap, same_value +from app.analysis.numbers import canon, to_fraction + +#: 稳健标准差 = 1.4826 × MAD（正态时等于标准差） +MAD_SCALE = Fraction(14826, 10000) +#: lag 的上限（周数据同比是 52） +_LAG_MAX = 260 + + +@dataclass +class Series: + snap: Snap + period: str + measure: str + grain: str + label: str + points: list[tuple[pd.Period, Fraction]] # 按期次排好、度量不为空的 + nulls: int + params: dict[str, Any] + + +def _load(raw: Mapping[str, Any], env: Env, used: list[str]) -> Series | Any: + """读出一条序列；算不了时直接交回 refused 的结论。""" + snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号") + period = snap.column(need(raw, "period", "期次列"), "期次列") + measure = snap.column(need(raw, "measure", "度量列"), "度量列") + grain = raw.get("grain") + if grain is not None and grain not in (*pd.GRAINS, "seq"): + raise CallError("bad_grain", "粒度只能写 day、week、month、quarter，或者不写（按期次值推断）") + params: dict[str, Any] = {"query": snap.ref, "period": period, "measure": measure} + label = label_for(env, raw.get("label"), snap, measure) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + criterion = _criterion(env, None, "期") + env.inputs.use(snap, "data", [period, measure]) + if snap.truncated: + return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，序列不完整", + criterion=criterion) + i_period = snap.index(period) + values = [row[i_period] if i_period < len(row) else None for row in snap.rows] + parsed = pd.parse(values, grain) + if parsed is None: + sample = "、".join(dict.fromkeys(ph.key_text([v]) for v in values[:3])) + return refuse(env, params, used, "period_unreadable", + f"期次列「{period}」的值认不出是哪一期（{sample}）：要日期、YYYY-MM、YYYY-Www、YYYY-Qn 或整数序号", + criterion=criterion) + items, grain = parsed + params["grain"] = grain + seen: set[int] = set() + points: list[tuple[pd.Period, Fraction]] = [] + nulls = 0 + for r, item in enumerate(items): + if item.ordinal in seen: + raise CallError("duplicate_rows", ph.say(f"{snap.ref} 里同一期有不止一行：在 SQL 里按 ", + column_label(env, snap, period), " 汇总，每期一行")) + seen.add(item.ordinal) + value = snap.number(r, measure, "度量列") + if value is None: + nulls += 1 + continue + points.append((item, value)) + points.sort(key=lambda p: p[0].ordinal) + return Series(snap, period, measure, grain, label, points, nulls, params) + + +def _criterion(env: Env, n: int | None, unit: str) -> str: + if env.op == "baseline": + return ph.criterion_baseline(env.settings, n, unit) + return ph.criterion_trend(env.settings, n, unit) + + +def _completeness(env: Env, series: Series, item: pd.Period) -> tuple[str, str | None]: + """本期齐不齐（同可比性检查的整期）：(pass / short / unknown, 不齐的那一句)。""" + series.params["today"] = today_param(env) + if item.start is None: + return "unknown", None + span = cmp.period_span("本期", series.snap, series.period, item, series.grain, series.measure, env.facts, env.book) + return cmp.span_coverage(span, env.today, env.book) # type: ignore[arg-type] + + +def _shown_period(item: pd.Period) -> Any: + return item.raw + + +# -------------------------------------------------------------------------- +# 波动基线 +# -------------------------------------------------------------------------- + + +def baseline(raw: Mapping[str, Any], env: Env) -> Any: + used = ["k_warn", "k_alert", "min_history", "max_history"] + loaded = _load(raw, env, used) + if not isinstance(loaded, Series): + return loaded + series, params = loaded, loaded.params + basis = raw.get("basis") or "change" + if basis not in ("change", "level"): + raise CallError("bad_basis", "比法只能写 change（和比较期比的变化率）或 level（值本身）") + lag = raw.get("lag", pd.DEFAULT_LAG[series.grain]) + if isinstance(lag, bool) or not isinstance(lag, int) or not 1 <= lag <= _LAG_MAX: + raise CallError("bad_lag", f"往前比几期要写 1 到 {_LAG_MAX} 的整数（同比：周数据 52、月数据 12、季度 4）") + params.update(basis=basis, lag=lag) + unit, per = pd.COUNT_WORD[series.grain], pd.PER_WORD[series.grain] + criterion = _criterion(env, None, unit) + if not series.points: + return refuse(env, params, used, "history_short", f"只有 0 期历史，至少要 {int(env.settings['min_history'])} 期", + criterion=criterion) + if raw.get("target") is not None: + found = next((p for p in series.points if same_value(raw["target"], p[0].raw)), None) + if found is None: + raise CallError("period_not_found", f"期次列「{series.period}」里没有「{raw['target']}」（或那一期为空）：" + "本期要写序列里有值的一期，不写就是最后一期") + else: + found = series.points[-1] + target, x_level = found + params["target"] = target.raw + caveats: list[dict[str, str]] = [] + status, why = _completeness(env, series, target) + if status == "short": + return refuse(env, params, used, "partial_target", f"{why}，和完整的{per}比没有意义", criterion=criterion) + if status == "unknown" and target.start is not None: + caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE)) + by_ordinal = {p.ordinal: v for p, v in series.points} + before = [(p, v) for p, v in series.points if p.ordinal < target.ordinal] + if basis == "change": + prev = by_ordinal.get(target.ordinal - lag) + if prev is None: + return refuse(env, params, used, "target_gap", + f"本期和它的比较期（往前第 {lag} {'期' if series.grain == 'seq' else unit}）之间缺期，没有比较期的值", + criterion=criterion) + if prev == 0: + return refuse(env, params, used, "zero_base", ph.say("比较期的", series.label, "为 0，算不了变化率"), + criterion=criterion) + x = (x_level - prev) / abs(prev) + history = [(p, (v - by_ordinal[p.ordinal - lag]) / abs(by_ordinal[p.ordinal - lag]), v) for p, v in before + if by_ordinal.get(p.ordinal - lag) not in (None, 0)] + else: + x = x_level + history = [(p, v, v) for p, v in before] + history = history[-int(env.settings["max_history"]):] + n = len(history) + criterion = _criterion(env, n, unit) + if n < int(env.settings["min_history"]): + return refuse(env, params, used, "history_short", f"只有 {n} 期历史，至少要 {int(env.settings['min_history'])} 期", + criterion=criterion) + hv = [h for _, h, _ in history] + m = stats.median(hv) + sigma = MAD_SCALE * stats.mad(hv, m) + k_warn, k_alert = env.threshold("k_warn"), env.threshold("k_alert") + if sigma == 0: + z = None + code = "within_band" if x == m else "far_outside_band" + else: + z = (x - m) / sigma + code = "within_band" if abs(z) < k_warn else "outside_band" if abs(z) < k_alert else "far_outside_band" + lo, hi = m - k_warn * sigma, m + k_warn * sigma + levels = [lv for _, _, lv in history] + if x_level > max(levels): + caveats.append(ph.caveat("record_high", ph.record_text(True, series.label, n, unit))) + elif x_level < min(levels): + caveats.append(ph.caveat("record_low", ph.record_text(False, series.label, n, unit))) + if series.nulls: + caveats.append(ph.caveat("null_values", ph.null_periods_text(series.nulls, series.label))) + current = ph.current_text(x, basis=basis, lag=lag, grain=series.grain) + if sigma == 0: + shown = ph.pct(m) if basis == "change" else ph.amount(m) + main = ph.baseline_flat_phrase(code, series.label, current, n, unit, per, shown) + else: + main = ph.baseline_phrase(code, series.label, current, n, unit, ph.band_text(lo, hi, basis=basis)) + rows = [[p.raw, canon(lv), canon(h) if basis == "change" else None, "历史"] for p, h, lv in history] + rows.append([target.raw, canon(x_level), canon(x) if basis == "change" else None, "本期"]) + table = {"columns": [series.period, "value", "change", "role"], + "labels": {series.period: column_label(env, series.snap, series.period), "value": series.label, + "change": "变化率", "role": "说明"}, + "rows": rows} + values = {"x": canon(x), "value": canon(x_level), "median": canon(m), "band_lo": canon(lo), "band_hi": canon(hi), + "z": canon(z) if z is not None else None, "rank": str(stats.rank_desc(x, hv)), "n_history": str(n), + "hist_min": canon(min(hv)), "hist_max": canon(max(hv))} + verdict = {"code": code, "subjects": [], "direction": sign_word(x - m)} + return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, + phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats) + + +# -------------------------------------------------------------------------- +# 趋势 +# -------------------------------------------------------------------------- + + +def run_length(values: list[Fraction]) -> tuple[int, str | None]: + """末尾连续同向：从最后一期往前数，相邻变化同号的有几个（变化为 0 断开）。""" + changes = [b - a for a, b in zip(values, values[1:])] + if not changes or changes[-1] == 0: + return 0, None + sign = changes[-1] > 0 + count = 0 + for c in reversed(changes): + if c == 0 or (c > 0) != sign: + break + count += 1 + return count, "up" if sign else "down" + + +def trend(raw: Mapping[str, Any], env: Env) -> Any: + used = ["trend_min_points", "trend_alpha", "max_history"] + loaded = _load(raw, env, used) + if not isinstance(loaded, Series): + return loaded + series, params = loaded, loaded.params + unit, per = pd.COUNT_WORD[series.grain], pd.PER_WORD[series.grain] + minimum = int(env.settings["trend_min_points"]) + criterion = _criterion(env, None, unit) + if not series.points: + return refuse(env, params, used, "history_short", f"只有 0 期，至少要 {minimum} 期", criterion=criterion) + last = series.points[-1][0] + caveats: list[dict[str, str]] = [] + status, why = _completeness(env, series, last) + if status == "short": + return refuse(env, params, used, "partial_target", f"{why}，和完整的{per}比没有意义", criterion=criterion) + if status == "unknown" and last.start is not None: + caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE)) + cap = int(env.settings["max_history"]) + 1 + tail = [series.points[-1]] + for point in reversed(series.points[:-1]): + if len(tail) >= cap or point[0].ordinal != tail[-1][0].ordinal - 1: + break + tail.append(point) + tail.reverse() + n = len(tail) + criterion = _criterion(env, n, unit) + if n < minimum: + gap = len(series.points) > n and len(tail) < cap + if gap: + return refuse(env, params, used, "series_gap", + f"最后连着的只有 {n} 期，再往前缺期（至少要连着的 {minimum} 期）", criterion=criterion) + return refuse(env, params, used, "history_short", f"只有 {n} 期，至少要 {minimum} 期", criterion=criterion) + ys = [v for _, v in tail] + xs = [Fraction(i) for i in range(n)] + slope = stats.theil_sen(xs, ys) + med = stats.median(ys) + rel = slope / abs(med) if med != 0 else None + mk = stats.mann_kendall(ys) + alpha = env.threshold("trend_alpha") + p = mk.p if isinstance(mk.p, Fraction) else to_fraction(mk.p) + if p is not None and p < alpha and slope > 0: + code = "rising" + elif p is not None and p < alpha and slope < 0: + code = "falling" + else: + code = "no_trend" + run, direction = run_length(ys) + if series.grain == "day": + caveats.append(ph.caveat("weekday_effect", ph.WEEKDAY_EFFECT)) + if series.nulls: + caveats.append(ph.caveat("null_values", ph.null_periods_text(series.nulls, series.label))) + slope_text = ph.pct(rel) if rel is not None else ph.amount(slope, signed=True) + main = ph.trend_phrase(code, series.label, n, unit, per, slope_text, + (run, direction) if run >= 3 and direction else None) # type: ignore[arg-type] + rows = [[p_.raw, canon(v), canon(v - tail[i - 1][1]) if i else None] for i, (p_, v) in enumerate(tail)] + table = {"columns": [series.period, "value", "change"], + "labels": {series.period: column_label(env, series.snap, series.period), "value": series.label, + "change": "比上一期"}, + "rows": rows} + values = {"slope": canon(slope), "rel_slope": canon(rel) if rel is not None else None, "p": canon(mk.p), + "s": str(mk.s), "n": str(n), "run_length": str(run), "run_direction": direction} + verdict = {"code": code, "subjects": [], "direction": "up" if code == "rising" else "down" if code == "falling" + else None} + return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, + phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats) + + +__all__ = ["MAD_SCALE", "baseline", "run_length", "trend"] diff --git a/backend/tests/analysis_samples.py b/backend/tests/analysis_samples.py new file mode 100644 index 00000000..01fae6b0 --- /dev/null +++ b/backend/tests/analysis_samples.py @@ -0,0 +1,127 @@ +"""分析算子测试用的样例：虚构零售库 shop（地区华东、华南、华北、西南，渠道线上、线下，销售额、订单数、退款），表名、列名、 +数字都是编的。analyze 跑一次 analysis.run，顺手用 recompute 复算一遍、核对逐字段相同（contract.same），所以每个用例都同时 +钉住「复算和运行逐字段相同」。""" +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from datetime import date +from typing import Any + +from app import analysis +from app.analysis.contract import CallError, effective_settings, same +from app.core.artifact_store import canonical_json, content_hash + +#: 测试里的「运行当天」 +TODAY = date(2026, 10, 21) + + +def snap(ref: str, sql: str, columns: Sequence[str], rows: Sequence[Sequence[Any]], *, + types: Mapping[str, str] | None = None, source: str = "shop", truncated: bool = False, + masked: Sequence[str] = (), **extra: Any) -> dict[str, Any]: + """一份输入快照（InputSnapshot 的形状）。artifact 按内容算，复算按它取回。""" + body = {"source": source, "sql": sql, "columns": list(columns), "rows": [list(r) for r in rows], + "truncated": truncated, **({"column_types": dict(types)} if types else {}), **extra} + return {"ref": ref, "artifact": content_hash(canonical_json(body)), "node_id": "q1", "masked": list(masked), **body} + + +def item(value: Any, status: str = "confirmed", source: str = "human") -> dict[str, Any]: + return {"value": value, "source": source, "status": status} + + +def notes(*, columns: Mapping[str, Mapping[str, Any]] | None = None, business_date: str | None = None) -> dict[str, Any]: + """一张表的目录 notes：columns 是 {列: {label / measure: 值或 item(...)}}。""" + out: dict[str, Any] = {} + if columns: + out["columns"] = {c: {k: v if isinstance(v, Mapping) else item(v) for k, v in spec.items()} + for c, spec in columns.items()} + if business_date: + out["business_date"] = item({"column": business_date, "rule": "下单日", "timezone": "Asia/Shanghai"}) + return out + + +#: orders 表的目录：地区、渠道、实收金额（流量）、退款单数 +ORDERS = notes(columns={"region": {"label": "地区"}, "channel": {"label": "渠道"}, + "amount": {"label": "销售额", "measure": "flow"}, + "refund_flag": {"label": "退款单数", "measure": "flow"}, + "order_date": {"label": "下单日"}}, + business_date="order_date") +CATALOGS = {"shop": {"orders": ORDERS}} + + +async def analyze(op: str, mode: str | None = None, *, snaps: Sequence[Mapping[str, Any]], today: date = TODAY, + catalogs: Mapping[str, Any] | None = None, probe: Any = None, facts: Any = None, + parents: Mapping[str, Mapping[str, Any]] | None = None, settings: Mapping[str, Any] | None = None, + hypothesis: str | None = None, check: bool = True, **params: Any) -> dict[str, Any]: + """跑一次 analysis.run；check 时再复算一遍，核对和封存的逐字段相同（hypothesis 除外）。""" + by_ref = {s["ref"]: s for s in snaps} + request: dict[str, Any] = {"op": op, "mode": mode, "params": params} + if hypothesis is not None: + request["hypothesis"] = hypothesis + context: dict[str, Any] = {"today": today, "catalogs": CATALOGS if catalogs is None else catalogs} + if probe is not None: + context["probe"] = probe + if facts is not None: + context["facts"] = facts + result = await analysis.run(request, by_ref.get, settings=effective_settings(settings), context=context, + parents=(parents or {}).get) + if check: + again = recomputed(result, snaps, parents) + assert same(result, again), _diff(result, again) + assert again["hypothesis"] == result["hypothesis"] + return dict(result) + + +def recomputed(result: Mapping[str, Any], snaps: Sequence[Mapping[str, Any]], + parents: Mapping[str, Mapping[str, Any]] | None = None) -> dict[str, Any]: + """按结论里记下的复算：快照按 artifact 取回（只交快照内容本身，不交入口加的出处），上一层结论按内容哈希取回。""" + by_artifact = {s["artifact"]: {k: v for k, v in s.items() if k not in ("ref", "artifact", "node_id", "masked")} + for s in snaps} + by_hash = {content_hash(canonical_json(r)): r for r in (parents or {}).values()} + return dict(analysis.recompute(result, by_artifact.get, load_result=by_hash.get)) # type: ignore[arg-type] + + +def _diff(a: Mapping[str, Any], b: Mapping[str, Any]) -> str: + keys = sorted(k for k in set(a) | set(b) if a.get(k) != b.get(k)) + return "；".join(f"{k}: {json.dumps(a.get(k), ensure_ascii=False)} ≠ {json.dumps(b.get(k), ensure_ascii=False)}" + for k in keys) + + +async def call_error(op: str, mode: str | None = None, **kwargs: Any) -> CallError: + """跑一次，期望是调用错误。""" + try: + await analyze(op, mode, check=False, **kwargs) + except CallError as e: + return e + raise AssertionError("应当是调用错误") + + +def fixed_prober(ranges: Mapping[tuple[str, str], tuple[str | None, str | None]] | None = None, *, + bounds: Mapping[str, str] | None = None, at: str = "2026-10-21T09:00:00") -> Any: + """测试用的探查器：ranges 是 {(表, 列): (最早, 最晚)}，bounds 是 {界的 SQL: 日期}；没写的查不到。""" + + class Prober: + def __init__(self) -> None: + self.calls: list[tuple[str, ...]] = [] + + async def __call__(self, source: str, table: str, column: str) -> dict[str, Any] | None: + self.calls.append((source, table, column)) + found = (ranges or {}).get((table, column)) + if found is None: + return None + return {"source": source, "table": table, "column": column, "min": found[0], "max": found[1], + "basis": "live", "at": at} + + async def bounds(self, source: str, expr: str) -> dict[str, Any] | None: + self.calls.append((source, expr)) + value = (bounds or {}).get(expr) + return None if value is None else {"source": source, "expr": expr, "value": value, "basis": "live", "at": at} + + def dialect(self, source: str) -> str | None: + return "sqlite" + + return Prober() + + +__all__ = ["CATALOGS", "ORDERS", "TODAY", "analyze", "call_error", "fixed_prober", "item", "notes", "recomputed", + "snap"] diff --git a/backend/tests/test_analysis_comparable.py b/backend/tests/test_analysis_comparable.py new file mode 100644 index 00000000..bf9bb3d8 --- /dev/null +++ b/backend/tests/test_analysis_comparable.py @@ -0,0 +1,182 @@ +"""可比性检查（app/analysis/comparable.py，细案 01 §6）：只差日期 / 差了别的筛选 / 结构不同、窗口不等（流量不可比、比率只提醒）、 +本期不完整、周末天数不同、探查失败退回剖析事实、都没有就核对不了。每个用例都顺带核对复算和运行逐字段相同（复算用记下的 +probes，不现查）。""" +from __future__ import annotations + +from analysis_samples import analyze, fixed_prober, snap +from app.analysis import comparable_reasons + +WINDOW = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= '{a}' AND order_date < '{b}'" +RANGE = {("orders", "order_date"): ("2025-01-01", "2026-10-20")} + + +def period(ref, a, b, sql=WINDOW, **kw): + return snap(ref, sql.format(a=a, b=b), ["gmv"], [[100]], types={"gmv": "number"}, **kw) + + +async def compare(base, target, *, probe=None, **params): + params = {"base_query": base["ref"], "target_query": target["ref"], **params} + return await analyze("comparable", snaps=[base, target], probe=probe, **params) + + +def checks(result): + return {r[0]: r[1] for r in result["table"]["rows"]} + + +async def test_two_weeks_that_differ_only_in_dates_are_comparable(): + result = await compare(period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19"), + probe=fixed_prober(RANGE), measure="gmv") + assert result["method"] == "comparable@1" and result["mode"] is None + assert result["verdict"] == {"code": "comparable", "subjects": [], "direction": None} + assert result["phrase"] == "两期可比" + assert checks(result) == {"同口径": "通过", "窗口长度": "通过", "整期": "通过", "星期构成": "通过", "数据问题": "通过"} + assert result["values"]["base_days"] == "7" and result["values"]["target_start"] == "2026-10-12" + assert result["values"]["window"] == "pass" and result["settings"] == {} + assert result["params"]["today"] == "2026-10-21" and result["params"]["measure"] == "gmv" + assert [p["column"] for p in result["probes"]] == ["order_date"] + assert result["table"]["columns"] == ["check", "status", "base", "target", "note"] + + +async def test_other_literals_differing_mean_two_segments_not_two_periods(): + sql = WINDOW + " AND region = '{r}'" + base = snap("Q1", sql.format(a="2026-10-05", b="2026-10-12", r="华南"), ["gmv"], [[100]]) + target = snap("Q2", sql.format(a="2026-10-12", b="2026-10-19", r="华东"), ["gmv"], [[100]]) + result = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv") + assert result["verdict"]["code"] == "incomparable" + assert result["phrase"] == "两期不可比：两边除了日期，筛选的还不一样" + assert comparable_reasons(result) == ["两边除了日期，筛选的还不一样"] + + +async def test_a_different_structure_says_where_it_differs(): + base = period("Q1", "2026-10-05", "2026-10-12") + target = period("Q2", "2026-10-12", "2026-10-19", sql=WINDOW + " AND status = 'paid'") + result = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv") + assert result["verdict"]["code"] == "incomparable" + assert result["phrase"] == "两期不可比：两份查询的口径结构不同（差在筛选条件）" + other = await compare(base, period("Q2", "2026-10-12", "2026-10-19", source="crm"), measure="gmv") + assert "两份查询来自不同的数据源" in other["phrase"] + + +async def test_unequal_windows_fail_for_flows_and_only_warn_otherwise(): + base, target = period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-15") + flow = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv") # SUM(amount)：目录标的流量 + assert flow["verdict"]["code"] == "incomparable" and checks(flow)["窗口长度"] == "不通过" + assert "两段的天数不同：7 天对 3 天" in flow["phrase"] + avg_sql = "SELECT AVG(amount) AS aov FROM orders WHERE order_date >= '{a}' AND order_date < '{b}'" + ratio = await compare(period("Q1", "2026-10-05", "2026-10-12", sql=avg_sql), + period("Q2", "2026-10-12", "2026-10-15", sql=avg_sql), probe=fixed_prober(RANGE), + measure="gmv") + assert ratio["verdict"]["code"] == "conditional" and checks(ratio)["窗口长度"] == "提醒" + unlabeled = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv", catalogs={}) + assert unlabeled["verdict"]["code"] == "conditional" + assert any(c["code"] == "measure_unlabeled" for c in unlabeled["caveats"]) + + +async def test_whole_months_count_as_the_same_length(): + result = await compare(period("Q1", "2026-08-01", "2026-09-01"), period("Q2", "2026-09-01", "2026-10-01"), + probe=fixed_prober(RANGE), measure="gmv") + assert result["verdict"]["code"] == "comparable" and result["values"]["base_days"] == "31" + + +async def test_an_unfinished_period_is_not_a_whole_period(): + result = await compare(period("Q1", "2026-10-12", "2026-10-19"), period("Q2", "2026-10-19", "2026-10-26"), + probe=fixed_prober(RANGE), measure="gmv") + assert result["verdict"]["code"] == "incomparable" + assert result["phrase"] == "两期不可比：本期数据只到 2026-10-20，还差 5 天" + late = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-16")}) + behind = await compare(period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19"), + probe=late, measure="gmv") + assert behind["phrase"] == "两期不可比：本期数据只到 2026-10-16，还差 2 天" + + +async def test_weekend_composition_of_short_windows(): + # 周一到周三 对 周五到周日 + result = await compare(period("Q1", "2026-10-05", "2026-10-08"), period("Q2", "2026-10-16", "2026-10-19"), + probe=fixed_prober(RANGE), measure="gmv") + assert result["verdict"]["code"] == "conditional" + assert result["phrase"] == "两期有条件可比：两段的周末天数不同：0 天对 2 天" + + +async def test_a_failed_probe_falls_back_to_profile_facts_then_says_it_cannot_check(): + facts = {"shop": {"orders": {"at": "2026-10-20T03:00:00", "date_ranges": [ + {"column": "order_date", "min": "2025-01-01", "max": "2026-10-19"}]}}} + base, target = period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19") + fallback = await compare(base, target, probe=fixed_prober({}), facts=facts, measure="gmv") + assert fallback["verdict"]["code"] == "comparable" and fallback["phrase"] == "两期可比" + assert fallback["probes"] == [{"source": "shop", "table": "orders", "column": "order_date", "min": "2025-01-01", + "max": "2026-10-19", "basis": "profile", "at": "2026-10-20"}] + nothing = await compare(base, target, measure="gmv") + assert nothing["verdict"]["code"] == "comparable" + assert nothing["phrase"] == "两期可比，整期核对不了" and nothing["probes"] == [] + + +async def test_bounds_computed_from_the_run_day_are_evaluated_in_the_database_and_recorded(): + sql = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', '{a}') AND order_date < date('now', '{b}')" + base = snap("Q1", sql.format(a="-16 days", b="-9 days"), ["gmv"], [[100]]) + target = snap("Q2", sql.format(a="-9 days", b="-2 days"), ["gmv"], [[90]]) + bounds = {"DATE('now', '-16 days')": "2026-10-05", "DATE('now', '-9 days')": "2026-10-12", + "DATE('now', '-2 days')": "2026-10-19"} + probe = fixed_prober(RANGE, bounds=bounds) + result = await compare(base, target, probe=probe, measure="gmv") + assert result["verdict"]["code"] == "comparable" + assert checks(result)["同口径"] == "通过" # 只差按运行当天现算的界 + assert result["values"]["base_start"] == "2026-10-05" and result["values"]["target_end"] == "2026-10-18" + exprs = [p["expr"] for p in result["probes"] if "expr" in p] + assert exprs == ["DATE('now', '-16 days')", "DATE('now', '-9 days')", "DATE('now', '-2 days')"] + assert all(p["basis"] == "live" for p in result["probes"]) + # 界算不出来：窗口核对不了，不判不可比 + blind = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv") + assert blind["verdict"]["code"] == "comparable" and "窗口长度" in blind["phrase"] + + +async def test_data_problems_on_the_snapshots(): + base = period("Q1", "2026-10-05", "2026-10-12") + empty = period("Q2", "2026-10-12", "2026-10-19", no_data=[{"columns": ["gmv"], "table": "orders", + "column": "order_date", "message": "没有数据"}]) + result = await compare(base, empty, probe=fixed_prober(RANGE), measure="gmv") + assert result["verdict"]["code"] == "incomparable" and "本期查询的区间里这张表没有数据" in result["phrase"] + checked = period("Q2", "2026-10-12", "2026-10-19", checks=[{"code": "fanout_sum", "level": "error", + "message": "关联订单明细后对订单金额求和，会重复计算"}]) + warned = await compare(base, checked, probe=fixed_prober(RANGE), measure="gmv") + assert warned["verdict"]["code"] == "conditional" + assert "本期查询的 SQL 检查有错误：关联订单明细后对订单金额求和，会重复计算" in warned["phrase"] + + +async def test_one_snapshot_grouped_by_period_is_naturally_the_same_query(): + sql = "SELECT order_date AS day, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-01' GROUP BY 1" + rows = [[f"2026-10-{d:02d}", 100] for d in range(1, 20)] + s = snap("Q1", sql, ["day", "gmv"], rows, types={"day": "text", "gmv": "number"}) + probe = fixed_prober(RANGE) + same_weekday = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="day", base="2026-10-05", + target="2026-10-12") + assert same_weekday["verdict"]["code"] == "comparable" and same_weekday["params"]["measure"] == "gmv" + assert same_weekday["params"]["grain"] == "day" + weekend = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="day", base="2026-10-09", + target="2026-10-10") + assert weekend["phrase"] == "两期有条件可比：两段的周末天数不同：0 天对 1 天" + # 期次有一截落在 SQL 的时间条件之外：基期被截掉了；本期（10 月）还没过完 + monthly_rows = [["2026-09-01", 100], ["2026-10-01", 90]] + month = snap("Q1", "SELECT DATE(order_date, 'start of month') AS month, SUM(amount) AS gmv FROM orders " + "WHERE order_date >= '2026-09-05' GROUP BY 1", ["month", "gmv"], monthly_rows) + partial = await analyze("comparable", snaps=[month], probe=probe, query="Q1", period="month", base="2026-09-01", + target="2026-10-01", measure="gmv") + assert partial["verdict"]["code"] == "incomparable" + assert "基期数据从 2026-09-05 才开始，缺前 4 天" in partial["phrase"] + assert "本期数据只到 2026-10-20，还差 11 天" in partial["phrase"] + + +async def test_reasons_are_split_one_sentence_each(): + result = await compare(period("Q1", "2026-10-05", "2026-10-08"), period("Q2", "2026-10-16", "2026-10-21"), + probe=fixed_prober(RANGE), measure="gmv") + reasons = comparable_reasons(result) + assert "两段的天数不同：3 天对 5 天" in reasons and "本期数据只到 2026-10-20，还差 0 天" not in reasons + assert all("；" not in r for r in reasons) + assert comparable_reasons({"table": None}) == [] + + +async def test_a_masked_date_column_is_neither_used_nor_probed(): + probe = fixed_prober(RANGE) + result = await compare(period("Q1", "2026-10-05", "2026-10-12", masked=["order_date"]), + period("Q2", "2026-10-12", "2026-10-19", masked=["order_date"]), probe=probe, measure="gmv") + assert probe.calls == [] and result["probes"] == [] + assert result["verdict"]["code"] == "comparable" and "窗口长度" in result["phrase"] diff --git a/backend/tests/test_analysis_decompose.py b/backend/tests/test_analysis_decompose.py new file mode 100644 index 00000000..8e34a692 --- /dev/null +++ b/backend/tests/test_analysis_decompose.py @@ -0,0 +1,420 @@ +"""变化拆解（app/analysis/decompose.py，细案 01 §2）：三种恒等式逐位成立、辛普森、抵消、不闭合、下钻闭合与超深、新出现 / +消失的段、比率当可加的调用错误……每个用例都顺带核对复算和运行逐字段相同（tests/analysis_samples.analyze）。""" +from __future__ import annotations + +import math +import random +from decimal import Decimal +from fractions import Fraction + +import pytest + +from analysis_samples import analyze, call_error, fixed_prober, notes, snap +from app.analysis.decompose import lmdi, ratio_effects + +SQL = "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week" +TYPES = {"region": "text", "week": "text", "gmv": "number"} +WEEKLY = [["华东", "2026-W41", 820], ["华东", "2026-W42", 700], ["华北", "2026-W41", 600], ["华北", "2026-W42", 560], + ["华南", "2026-W41", 700], ["华南", "2026-W42", 666], ["西南", "2026-W41", 100], ["西南", "2026-W42", 100]] + + +def weekly(rows=WEEKLY, **kw): + return snap("Q2", SQL, ["region", "week", "gmv"], rows, types=TYPES, **kw) + + +async def additive(rows=WEEKLY, *, snaps=None, **params): + params = {"query": "Q2", "by": ["region"], "period": "week", "base": "2026-W41", "target": "2026-W42", + "measure": "gmv", **params} + return await analyze("decompose", "additive", snaps=snaps or [weekly(rows)], **params) + + +def _rows(base: dict, target: dict) -> list[list]: + return [[k, "2026-W41", v] for k, v in base.items()] + [[k, "2026-W42", v] for k, v in target.items()] + + +# -------------------------------------------------------------------------- +# additive +# -------------------------------------------------------------------------- + + +async def test_a_driver_is_named_with_its_share_and_the_whole_shape_is_filled(): + result = await additive(label="销售额", hypothesis="华东拖累") + assert result["method"] == "decompose.additive@1" and result["status"] == "ok" + assert result["verdict"] == {"code": "driver", "subjects": [["华东"]], "direction": "down"} + assert result["phrase"] == "销售额下降 194（-8.7%），主要来自华东（-120，占 61.9%）" + assert result["values"]["delta_total"] == "-194" and result["values"]["top_key"] == "华东" + assert result["values"]["top_share"] == "0.618556701031" and result["values"]["segments"] == "4" + table = result["table"] + assert table["columns"] == ["region", "base", "target", "delta", "pct", "share", "flag"] + assert table["labels"]["region"] == "地区" and table["labels"]["share"] == "占变化的比重" + assert [r[0] for r in table["rows"]] == ["华东", "华北", "华南", "西南"] # 按变化的绝对值从大到小 + assert result["settings"] == {"main_share": 0.5, "concentration": 0.8, "offset_ratio": 0.25, "flat_pct": 0.005, + "table_top_k": 10} + assert result["inputs"] == [{"ref": "Q2", "artifact": weekly()["artifact"], "role": "data", + "columns": ["region", "week", "gmv"], "rows": "all"}] + assert result["hypothesis"] == "华东拖累" and result["probes"] == [] + assert result["criterion"].endswith("以各段之和为总数") + assert {c["code"] for c in result["caveats"]} == {"headline_basis"} + assert result["params"]["catalog"] == {"labels": {"region": "地区"}, "measures": {"gmv": "flow"}} + + +async def test_the_segments_add_up_to_the_total_digit_for_digit(): + rng = random.Random(7) + regions = ["华东", "华南", "华北", "西南", "东北", "西北"] + for _ in range(20): + base = {r: round(rng.uniform(0, 5000), rng.choice([0, 2, 3])) for r in regions} + target = {r: round(rng.uniform(0, 5000), rng.choice([0, 2, 3])) for r in regions} + result = await additive(_rows(base, target)) + rows = result["table"]["rows"] + assert sum(Decimal(r[3]) for r in rows) == Decimal(result["values"]["delta_total"]) + assert sum(Decimal(r[1]) for r in rows) == Decimal(result["values"]["base_total"]) + assert sum(Decimal(r[2]) for r in rows) == Decimal(result["values"]["target_total"]) + assert Decimal(result["values"]["target_total"]) - Decimal(result["values"]["base_total"]) == \ + Decimal(result["values"]["delta_total"]) + assert result["values"]["gross_up"] and result["values"]["gross_down"] + + +async def test_concentrated_and_dispersed(): + base = {f"店{i}": 1000 for i in range(10)} + target = {**base, "店0": 1450, "店1": 1400, "店2": 1050, "店3": 1050, "店4": 1050} + result = await additive(_rows(base, target), by="region") + assert result["verdict"]["code"] == "concentrated" + assert result["verdict"]["subjects"] == [["店0"], ["店1"]] + assert result["values"]["top_k"] == "2" and result["values"]["top_k_share"] == "0.85" + assert "集中在店0、店1（合计占 85.0%）" in result["phrase"] + spread = await additive(_rows({r: 1000 for r in "甲乙丙丁戊"}, {r: 1100 for r in "甲乙丙丁戊"})) + assert spread["verdict"] == {"code": "dispersed", "subjects": [], "direction": "up"} + assert "分散在各地区，没有哪一段占到一半" in spread["phrase"] + + +async def test_offsetting_segments_are_said_and_a_share_can_pass_one_hundred_percent(): + result = await additive(_rows({"华东": 1000, "华北": 500, "华南": 500}, {"华东": 850, "华北": 540, "华南": 490})) + assert result["values"]["delta_total"] == "-120" + assert result["verdict"]["code"] == "driver" and result["values"]["top_share"] == "1.25" + offset = next(c for c in result["caveats"] if c["code"] == "offset") + assert offset["text"] == "下降的段合计 -160，上升的段合计 +40，互相抵消了一部分" + assert result["phrase"] == "销售额下降 120（-6.0%），主要来自华东（-150，占 125.0%）；下降的段合计 -160，上升的段合计 +40，互相抵消了一部分" + + +async def test_flat_with_offsets_says_what_moved_inside(): + result = await additive(_rows({"华东": 1000, "华北": 1000, "华南": 220}, {"华东": 1120, "华北": 882, "华南": 220})) + assert result["verdict"] == {"code": "flat", "subjects": [], "direction": None} + assert result["phrase"] == "销售额基本持平（+0.1%），内部此消彼长：华东 +120、华北 -118" + + +async def test_a_headline_that_does_not_close_stops_the_concentration_verdict(): + head = snap("Q1", "SELECT week, SUM(amount) AS gmv FROM orders GROUP BY week", ["week", "gmv"], + [["2026-W42", 2050], ["2026-W41", 2220]], types={"week": "text", "gmv": "number"}) + result = await additive(snaps=[weekly(), head], headline={"base": "Q1.r1.gmv", "target": "Q1.r0.gmv"}) + assert result["verdict"] == {"code": "unclosed", "subjects": [], "direction": None} + assert result["values"]["gap"] == "24" and result["values"]["top_share"] is None + assert all(r[5] is None for r in result["table"]["rows"]) # 不给占比 + assert result["phrase"] == "按地区拆开后各段之和和总数对不上（差 24），不判断变化来自哪里" + assert [i["role"] for i in result["inputs"]] == ["headline_base", "headline_target", "data"] + assert result["inputs"][0] == {"ref": "Q1.r1.gmv", "artifact": head["artifact"], "role": "headline_base", + "row": 1, "column": "gmv", "value": "2220"} + assert "各段之和要和总数对得上" in result["criterion"] + + +async def test_a_rounded_headline_closes_within_its_written_precision(): + rows = [["华东", "2026-W41", "820.25"], ["华东", "2026-W42", "700.12"], ["华北", "2026-W41", "1399.9"], + ["华北", "2026-W42", "1325.84"]] + for written, closes in (("2220.2", True), ("2220", True), ("2221", False)): + head = snap("Q1", "SELECT SUM(amount) AS gmv FROM orders", ["gmv_base", "gmv"], [[written, "2025.96"]]) + result = await additive(rows, snaps=[weekly(rows), head], headline=["Q1.r0.gmv_base", "Q1.r0.gmv"]) + assert (result["verdict"]["code"] != "unclosed") is closes, written + + +async def test_drilling_down_checks_closure_against_the_parent_segment(): + top = await additive() + province = snap("Q4", "SELECT province, week, SUM(amount) AS gmv FROM orders WHERE region = '华东' GROUP BY 1, 2", + ["province", "week", "gmv"], + [["沪", "2026-W41", 500], ["沪", "2026-W42", 410], ["苏", "2026-W41", 320], ["苏", "2026-W42", 290]]) + params = {"query": "Q4", "by": ["province"], "period": "week", "base": "2026-W41", "target": "2026-W42", + "measure": "gmv", "parent": {"of": "A3", "key": ["华东"]}} + child = await analyze("decompose", "additive", snaps=[province], parents={"A3": top}, **params) + assert child["verdict"]["code"] == "driver" and child["verdict"]["subjects"] == [["沪"]] + assert child["params"]["depth"] == 1 and child["params"]["parent"] == {"of": "A3", "key": ["华东"]} + assert child["inputs"][0]["role"] == "parent" and child["inputs"][0]["ref"] == "A3" + assert "max_drill_depth" in child["settings"] + assert {c["code"] for c in child["caveats"]} == set() # 有 parent 就不是「以各段之和为总数」 + short = snap("Q4", province["sql"], ["province", "week", "gmv"], + [["沪", "2026-W41", 500], ["沪", "2026-W42", 410], ["苏", "2026-W41", 300], ["苏", "2026-W42", 290]]) + unclosed = await analyze("decompose", "additive", snaps=[short], parents={"A3": top}, **params) + assert unclosed["verdict"]["code"] == "unclosed" and unclosed["values"]["gap"] == "20" + # 超深：上一层已经是第 1 层，最多 1 层 + error = await call_error("decompose", "additive", snaps=[province], parents={"A3": child}, + settings={"max_drill_depth": 1}, **{**params, "parent": {"of": "A3", "key": ["沪"]}}) + assert error.code == "too_deep" and "最多 1 层" in error.message + missing = await call_error("decompose", "additive", snaps=[province], parents={"A3": top}, + **{**params, "parent": {"of": "A3", "key": ["东北"]}}) + assert missing.code == "parent_key_missing" + absent = await call_error("decompose", "additive", snaps=[province], **params) + assert absent.code == "parent_missing" + + +async def test_new_and_vanished_segments_and_empty_cells_are_flagged(): + rows = [["华东", "2026-W41", 800], ["华东", "2026-W42", 700], ["西南", "2026-W42", 90], + ["东北", "2026-W41", 40], ["华北", "2026-W41", None], ["华北", "2026-W42", 30]] + result = await additive(rows) + flags = {r[0]: r[6] for r in result["table"]["rows"]} + assert flags == {"华东": None, "西南": "新出现", "东北": "消失", "华北": "为空"} + null = next(c for c in result["caveats"] if c["code"] == "null_values") + assert null["text"] == "华北的销售额为空，按 0 算" + assert next(r for r in result["table"]["rows"] if r[0] == "西南")[4] is None # 基期为 0 不给变化率 + + +async def test_two_dimension_keys(): + sql = "SELECT region, channel, week, SUM(amount) AS gmv FROM orders GROUP BY 1, 2, 3" + rows = [["华东", "线上", "W1", 500], ["华东", "线上", "W2", 300], ["华东", "线下", "W1", 300], ["华东", "线下", "W2", 310], + ["华北", "线上", "W1", 200], ["华北", "线上", "W2", 205]] + s = snap("Q2", sql, ["region", "channel", "week", "gmv"], rows) + result = await analyze("decompose", "additive", snaps=[s], query="Q2", by=["region", "channel"], period="week", + base="W1", target="W2", measure="gmv") + assert result["verdict"]["subjects"] == [["华东", "线上"]] + assert "主要来自华东·线上" in result["phrase"] + + +async def test_ratio_and_non_additive_measures_are_call_errors(): + avg = snap("Q2", "SELECT region, week, AVG(amount) AS aov FROM orders GROUP BY 1, 2", ["region", "week", "aov"], WEEKLY) + error = await call_error("decompose", "additive", snaps=[avg], query="Q2", by="region", period="week", + base="2026-W41", target="2026-W42", measure="aov") + assert error.code == "ratio_as_additive" and "改用比率拆解" in error.message + rate_sql = "SELECT region, week, SUM(refund_rate) AS r FROM orders GROUP BY 1, 2" + labeled = {"shop": {"orders": notes(columns={"refund_rate": {"measure": "ratio"}})}} + error = await call_error("decompose", "additive", snaps=[snap("Q2", rate_sql, ["region", "week", "r"], WEEKLY)], + catalogs=labeled, query="Q2", by="region", period="week", base="2026-W41", + target="2026-W42", measure="r") + assert error.code == "ratio_as_additive" + # 只是推断（没人确认）的照算，写附带说明 + guessed = {"shop": {"orders": notes(columns={"refund_rate": {"measure": {"value": "ratio", "source": "llm", + "status": "proposed"}}})}} + result = await analyze("decompose", "additive", snaps=[snap("Q2", rate_sql, ["region", "week", "r"], WEEKLY)], + catalogs=guessed, query="Q2", by="region", period="week", base="2026-W41", + target="2026-W42", measure="r") + note = next(c for c in result["caveats"] if c["code"] == "measure_unlabeled") + assert note["text"] == "目录里「r」是比率只是推断，没人确认，按可以按段相加处理" + unlabeled = await additive(snaps=[weekly()], catalogs={}) + assert any(c["code"] == "measure_unlabeled" and "目录没标「gmv」的度量类型" in c["text"] for c in unlabeled["caveats"]) + + +async def test_call_errors_tell_the_agent_how_to_fix_the_call(): + dup = WEEKLY + [["华东", "2026-W42", 5]] + error = await call_error("decompose", "additive", snaps=[weekly(dup)], query="Q2", by="region", period="week", + base="2026-W41", target="2026-W42", measure="gmv") + assert error.code == "duplicate_rows" and "在 SQL 里按 地区、week 汇总" in error.message + masked = await call_error("decompose", "additive", snaps=[weekly(masked=["Region"])], query="Q2", by="region", + period="week", base="2026-W41", target="2026-W42", measure="gmv") + assert masked.code == "masked_column" + missing = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="region", period="week", + base="2026-W40", target="2026-W42", measure="gmv") + assert missing.code == "period_not_found" and "2026-W41" in missing.message + column = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="area", period="week", + base="2026-W41", target="2026-W42", measure="gmv") + assert column.code == "unknown_column" and "region、week、gmv" in column.message + text = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="gmv", period="week", + base="2026-W41", target="2026-W42", measure="region") + assert text.code == "not_numeric" + unknown = await call_error("decompose", "additive", snaps=[weekly()], query="Q9", by="region", period="week", + base="2026-W41", target="2026-W42", measure="gmv") + assert unknown.code == "unknown_query" + same_column = await call_error("decompose", "additive", snaps=[weekly()], query="Q2", by="week", period="week", + base="2026-W41", target="2026-W42", measure="gmv") + assert same_column.code == "bad_by" + + +async def test_truncated_snapshots_and_single_segments_are_refused(): + result = await additive(snaps=[weekly(truncated=True)]) + assert result["status"] == "refused" and result["reason_code"] == "truncated" + assert result["reason"] == "Q2 只取回了前 8 行，各段之和不是完整的和" + assert result["phrase"] == "变化拆解没有做成：Q2 只取回了前 8 行，各段之和不是完整的和" + assert result["verdict"] is None and result["table"] is None and result["values"] == {} + one = await additive([["华东", "2026-W41", 1], ["华东", "2026-W42", 2]]) + assert one["reason_code"] == "too_few_segments" + + +async def test_two_snapshots_check_comparability_first(): + base = snap("Q1", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' AND " + "order_date < '2026-10-12' GROUP BY region", ["region", "gmv"], + [["华东", 820], ["华北", 600], ["华南", 700]]) + target = snap("Q2", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-12' AND " + "order_date < '2026-10-19' GROUP BY region", ["region", "gmv"], + [["华东", 700], ["华北", 560], ["华南", 666]]) + probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")}) + result = await analyze("decompose", "additive", snaps=[base, target], probe=probe, base_query="Q1", + target_query="Q2", by="region", measure="gmv") + assert result["verdict"]["code"] == "driver" and result["params"]["today"] == "2026-10-21" + assert not any(c["code"] == "comparability" for c in result["caveats"]) + assert [i["role"] for i in result["inputs"]] == ["base", "target"] + assert result["probes"] == [{"source": "shop", "table": "orders", "column": "order_date", "min": "2025-01-01", + "max": "2026-10-20", "basis": "live", "at": "2026-10-21T09:00:00"}] + # 本期没过完：照样拆，判定后附上可比性结论 + partial = snap("Q2", target["sql"].replace("'2026-10-19'", "'2026-10-25'"), ["region", "gmv"], target["rows"]) + result = await analyze("decompose", "additive", snaps=[base, partial], probe=probe, base_query="Q1", + target_query="Q2", by="region", measure="gmv") + note = next(c for c in result["caveats"] if c["code"] == "comparability") + assert note["text"].startswith("两期不可比：") and "本期数据只到 2026-10-20，还差 4 天" in note["text"] + assert result["verdict"]["code"] == "driver" and note["text"] in result["phrase"] + # 口径结构不同：不拆 + other = snap("Q2", "SELECT region, SUM(amount) AS gmv FROM orders JOIN refunds USING (order_id) WHERE order_date " + ">= '2026-10-12' GROUP BY region", ["region", "gmv"], target["rows"]) + refused = await analyze("decompose", "additive", snaps=[base, other], probe=probe, base_query="Q1", + target_query="Q2", by="region", measure="gmv") + assert refused["reason_code"] == "incomparable_sides" and refused["reason"].startswith("两份查询的口径结构不同") + + +# -------------------------------------------------------------------------- +# ratio +# -------------------------------------------------------------------------- + +RATIO_SQL = "SELECT channel, week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY 1, 2" + + +def _ratio_snap(rows): + return snap("Q1", RATIO_SQL, ["channel", "week", "refunds", "orders"], rows) + + +async def ratio(rows, **params): + params = {"query": "Q1", "by": "channel", "period": "week", "base": "W41", "target": "W42", + "numerator": "refunds", "denominator": "orders", **params} + return await analyze("decompose", "ratio", snaps=[_ratio_snap(rows)], **params) + + +def test_mix_plus_rate_effects_equal_the_ratio_change_exactly(): + rng = random.Random(11) + for _ in range(200): + segs = [] + for _ in range(rng.randint(2, 7)): + d_b, d_t = rng.choice([0, rng.randint(1, 5000)]), rng.randint(1, 5000) + segs.append((Fraction(rng.randint(0, d_b)) if d_b else Fraction(0), Fraction(d_b), + Fraction(rng.randint(0, d_t)), Fraction(d_t))) + if sum(s[1] for s in segs) == 0: + continue + R, R2, effects = ratio_effects(segs) + assert sum((e[4] for e in effects), Fraction(0)) + sum((e[5] for e in effects), Fraction(0)) == R2 - R + assert sum((e[0] for e in effects), Fraction(0)) == 1 == sum((e[1] for e in effects), Fraction(0)) + + +async def test_simpson_each_channel_improves_but_the_mix_makes_the_whole_worse(): + rows = [["线上", "W41", 50, 500], ["线上", "W42", 90, 1000], ["线下", "W41", 10, 1000], ["线下", "W42", 4, 500]] + result = await ratio(rows, label="退款率") + assert result["verdict"] == {"code": "mix_driven", "subjects": [["线上"]], "direction": "up"} + assert result["phrase"].startswith("退款率上升 2.3 个百分点（4.0% 到 6.3%），主要来自结构变化：线上占比从 33.3% 到 66.7%") + simpson = next(c for c in result["caveats"] if c["code"] == "simpson") + assert simpson["text"] == "各段的退款率大多在下降，整体却上升：变化来自结构，线上占比从 33.3% 升到 66.7%" + values = result["values"] + assert Fraction(values["mix_total"]) + Fraction(values["rate_total"]) == pytest.approx(Fraction(values["delta"])) + assert result["table"]["columns"][:5] == ["channel", "n_base", "n_target", "d_base", "d_target"] + + +async def test_rate_driven_both_and_small_denominators(): + rows = [["线上", "W41", 50, 1000], ["线上", "W42", 80, 1000], ["线下", "W41", 20, 1000], ["线下", "W42", 26, 1000]] + result = await ratio(rows, label="退款率") + assert result["verdict"]["code"] == "rate_driven" and result["verdict"]["subjects"] == [["线上"]] + assert result["phrase"] == "退款率上升 1.8 个百分点（3.5% 到 5.3%），主要来自各段自身的变化，线上影响最大" + tiny = [["线上", "W41", 2, 20], ["线上", "W42", 3, 25], ["线下", "W41", 20, 1000], ["线下", "W42", 30, 1000]] + small = await ratio(tiny, label="退款率") + note = next(c for c in small["caveats"] if c["code"] == "small_denominator") + assert note["text"] == "线上的 orders 不到 30，比率波动大，不参与多数判断" + assert "分母小" in next(r for r in small["table"]["rows"] if r[0] == "线上")[-1] + + +async def test_ratio_refusals(): + zero = [["线上", "W41", 0, 0], ["线上", "W42", 3, 100], ["线下", "W41", 0, 0], ["线下", "W42", 2, 100]] + result = await ratio(zero) + assert result["reason_code"] == "zero_base" and result["reason"] == "基期的 orders 合计为 0，算不了比率" + orphan = [["线上", "W41", 5, 0], ["线上", "W42", 3, 100], ["线下", "W41", 1, 100], ["线下", "W42", 2, 100]] + assert (await ratio(orphan))["reason_code"] == "zero_base" + + +# -------------------------------------------------------------------------- +# factor +# -------------------------------------------------------------------------- + + +def _factor_snap(rows): + return snap("Q1", "SELECT week, SUM(amount) AS gmv, COUNT(*) AS orders, ROUND(AVG(amount), 2) AS aov FROM orders " + "GROUP BY week", ["week", "gmv", "orders", "aov"], rows) + + +async def factor(rows, **params): + params = {"query": "Q1", "period": "week", "base": "W41", "target": "W42", "product": "gmv", + "factors": ["orders", "aov"], "label": "销售额", **params} + return await analyze("decompose", "factor", snaps=[_factor_snap(rows)], **params) + + +def test_lmdi_contributions_add_up_to_the_change_within_float_error(): + rng = random.Random(3) + for _ in range(200): + k = rng.randint(2, 4) + base = [rng.uniform(0.5, 500) for _ in range(k)] + target = [b * rng.uniform(0.5, 1.8) for b in base] + m_b = math.prod(base) * (1 + rng.uniform(-1e-6, 1e-6)) # 取整过的因子：乘积差一点点 + m_t = math.prod(target) * (1 + rng.uniform(-1e-6, 1e-6)) + contrib = lmdi(base, target, m_b, m_t) + assert math.fsum(contrib) == pytest.approx(m_t - m_b, rel=1e-9, abs=1e-9 * max(m_b, m_t)) + same = lmdi([2.0, 3.0], [2.0, 3.0], 6.0, 6.0) + assert same == [0.0, 0.0] + + +async def test_factor_driver_with_rounded_factors(): + rows = [["W41", 1000, 137, "7.30"], ["W42", 1320, 160, "8.25"]] + result = await factor(rows) + assert result["verdict"]["code"] in ("driver", "both") + contrib = [Fraction(r[4]) for r in result["table"]["rows"]] + assert float(sum(contrib)) == pytest.approx(320, rel=1e-9) + clean = await factor([["W41", "1000", 100, "10"], ["W42", "1320", 120, "11"]]) + assert clean["verdict"] == {"code": "driver", "subjects": [["orders"]], "direction": "up"} + assert clean["phrase"].startswith("销售额上升 320（+32.0%），主要来自 orders 的变化（贡献 +") + assert clean["table"]["columns"] == ["factor", "base", "target", "pct", "contrib", "share"] + + +async def test_factor_call_errors_and_refusals(): + # 客单价写到两位小数：容差是它的半个末位（加上订单数的），差 1.5% 不是取整造成的 + error = await call_error("decompose", "factor", snaps=[_factor_snap([["W41", 1000, 100, "10.00"], + ["W42", 1300, 120, "11.00"]])], + query="Q1", period="week", base="W41", target="W42", product="gmv", + factors=["orders", "aov"]) + assert error.code == "not_product" and "本期差" in error.message + negative = await factor([["W41", 0, 0, 10], ["W42", 1320, 120, 11]]) + assert negative["reason_code"] == "non_positive" + flat = await factor([["W41", 1000, 100, 10], ["W42", 1000, 125, 8]]) + assert flat["verdict"]["code"] == "flat" + few = await call_error("decompose", "factor", snaps=[_factor_snap([["W41", 1, 1, 1], ["W42", 1, 1, 1]])], + query="Q1", period="week", base="W41", target="W42", product="gmv", factors=["orders"]) + assert few.code == "bad_factors" + + +async def test_ratio_headline_closure_and_drilling_into_a_channel(): + rows = [["线上", "W41", 50, 1000], ["线上", "W42", 80, 1000], ["线下", "W41", 20, 1000], ["线下", "W42", 26, 1000]] + head = snap("Q9", "SELECT week, SUM(refund_flag) * 1.0 / COUNT(*) AS rate FROM orders GROUP BY week", + ["week", "rate"], [["W41", "0.035"], ["W42", "0.053"]]) + params = {"query": "Q1", "by": "channel", "period": "week", "base": "W41", "target": "W42", + "numerator": "refunds", "denominator": "orders", "label": "退款率"} + closed = await analyze("decompose", "ratio", snaps=[_ratio_snap(rows), head], + headline={"base": "Q9.r0.rate", "target": "Q9.r1.rate"}, **params) + assert closed["verdict"]["code"] == "rate_driven" + off = snap("Q9", head["sql"], ["week", "rate"], [["W41", "0.035"], ["W42", "0.06"]]) + unclosed = await analyze("decompose", "ratio", snaps=[_ratio_snap(rows), off], + headline={"base": "Q9.r0.rate", "target": "Q9.r1.rate"}, **params) + assert unclosed["verdict"]["code"] == "unclosed" + assert unclosed["phrase"].startswith("按渠道拆开后各段合起来的比率和总数对不上") + # 线上按地区下钻：各地区的分子、分母之和要等于上一层线上那一段的 + region = snap("Q2", "SELECT region, week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders " + "WHERE channel = '线上' GROUP BY 1, 2", ["region", "week", "refunds", "orders"], + [["华东", "W41", 30, 600], ["华东", "W42", 55, 600], ["华北", "W41", 20, 400], ["华北", "W42", 25, 400]]) + child = await analyze("decompose", "ratio", snaps=[region], parents={"A1": closed}, + **{**params, "query": "Q2", "by": "region", "parent": {"of": "A1", "key": ["线上"]}}) + assert child["verdict"]["code"] != "unclosed" and child["params"]["depth"] == 1 + wrong = await call_error("decompose", "additive", snaps=[weekly()], parents={"A1": closed}, query="Q2", + by="region", period="week", base="2026-W41", target="2026-W42", measure="gmv", + parent={"of": "A1", "key": ["线上"]}) + assert wrong.code == "parent_not_decompose" + + +async def test_the_same_query_cannot_be_both_periods(): + error = await call_error("decompose", "additive", snaps=[weekly()], base_query="Q2", target_query="Q2", + by="region", measure="gmv") + assert error.code == "same_query" + error = await call_error("comparable", snaps=[weekly()], base_query="Q2", target_query="Q2") + assert error.code == "same_query" diff --git a/backend/tests/test_analysis_differ.py b/backend/tests/test_analysis_differ.py new file mode 100644 index 00000000..b8120d7b --- /dev/null +++ b/backend/tests/test_analysis_differ.py @@ -0,0 +1,138 @@ +"""差异检验（app/analysis/differ.py，细案 01 §5）：比率小样本走 Fisher、均值 Welch、Holm、rows 敏感性翻转。每个用例都顺带核对 +复算和运行逐字段相同。""" +from __future__ import annotations + +import math +from fractions import Fraction + +import pytest + +from analysis_samples import analyze, call_error, snap +from app.analysis import stats + +RATE_SQL = "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region" + + +def rates(rows): + return snap("Q1", RATE_SQL, ["region", "refunds", "orders"], rows) + + +async def rate(rows, **params): + params = {"query": "Q1", "by": "region", "numerator": "refunds", "denominator": "orders", "label": "退款率", **params} + return await analyze("differ", "rate", snaps=[rates(rows)], **params) + + +async def test_small_samples_use_fisher_and_say_it_may_be_noise(): + result = await rate([["西南", 2, 40], ["华东", 5, 230]]) + assert result["method"] == "differ.rate@1" + assert result["verdict"] == {"code": "no_evidence", "subjects": [["西南"], ["华东"]], "direction": None} + assert result["values"]["test"] == "fisher" and result["values"]["n_a"] == "40" and result["values"]["n_b"] == "230" + assert Fraction(result["values"]["p"]) == pytest.approx(stats.fisher_exact(2, 38, 5, 225), rel=1e-9) + assert result["phrase"] == ("西南和华东的退款率差异可能只是样本波动（样本量 40、230）；样本少（有格子的期望频数不到 5），" + "用了精确检验") + assert "p 值" not in result["phrase"] and "显著" not in result["phrase"] + assert result["settings"] == {"differ_alpha": 0.05, "min_expected": 5} + + +async def test_large_samples_use_the_pooled_z_test(): + result = await rate([["西南", 80, 1000], ["华东", 40, 1000]]) + assert result["verdict"] == {"code": "differs", "subjects": [["西南"], ["华东"]], "direction": "up"} + z, p = stats.pooled_z(80, 1000, 40, 1000) + assert Fraction(result["values"]["p"]) == pytest.approx(p, rel=1e-9) and result["values"]["test"] == "z" + low, high = stats.newcombe(80, 1000, 40, 1000) + assert Fraction(result["values"]["ci_lo"]) == pytest.approx(low, rel=1e-9) + assert result["values"]["a"] == "0.08" and result["values"]["diff"] == "0.04" + assert result["phrase"] == "西南的退款率（8.0%）高于华东（4.0%），差异站得住" + lower = await rate([["西南", 80, 1000], ["华东", 40, 1000]], pairs=[["华东", "西南"]]) + assert lower["verdict"]["direction"] == "down" and lower["phrase"].startswith("华东的退款率（4.0%）低于西南（8.0%）") + + +async def test_several_groups_each_against_the_rest_with_holm(): + rows = [["西南", 80, 1000], ["华东", 40, 1000], ["华南", 42, 1000], ["华北", 38, 1000]] + result = await rate(rows) + table = result["table"] + assert [r[0] for r in table["rows"]] == ["西南", "华东", "华南", "华北"] + assert table["rows"][0][1] == "其余各地区" + raw = [Fraction(r[10]) for r in table["rows"]] + adjusted = [Fraction(r[11]) for r in table["rows"]] + assert all(a >= p for a, p in zip(adjusted, raw)) + assert adjusted == pytest.approx(stats.holm([float(p) for p in raw]), rel=1e-9) + assert result["verdict"]["code"] == "differs" and result["verdict"]["subjects"] == [["西南"], ["其余各地区"]] + assert any(c["code"] == "multiple_comparisons" and c["text"] == "比了 4 对，已按次数校正" for c in result["caveats"]) + assert result["phrase"] == "比了 4 对，1 对差异站得住：西南的退款率（8.0%）高于其余各地区（4.0%）" + ref = await rate(rows, reference="华东") + assert [r[0] for r in ref["table"]["rows"]] == ["西南", "华南", "华北"] and ref["params"]["reference"] == "华东" + + +async def test_rate_call_errors(): + assert (await call_error("differ", "rate", snaps=[rates([["西南", 5, 4], ["华东", 1, 9]])], query="Q1", by="region", + numerator="refunds", denominator="orders")).code == "not_count" + assert (await call_error("differ", "rate", snaps=[rates([["西南", 1.5, 4], ["华东", 1, 9]])], query="Q1", + by="region", numerator="refunds", denominator="orders")).code == "not_count" + unknown = await call_error("differ", "rate", snaps=[rates([["西南", 1, 4], ["华东", 1, 9]])], query="Q1", + by="region", numerator="refunds", denominator="orders", pairs=[["西南", "东北"]]) + assert unknown.code == "unknown_group" and "西南、华东" in unknown.message + dup = await call_error("differ", "rate", snaps=[rates([["西南", 1, 4], ["西南", 1, 9]])], query="Q1", by="region", + numerator="refunds", denominator="orders") + assert dup.code == "duplicate_rows" + + +async def test_means_from_sums_and_squares_use_welch(): + sql = "SELECT channel, SUM(amount) AS s, SUM(amount * amount) AS ss, COUNT(*) AS n FROM orders GROUP BY channel" + rows = [["线上", 5000, 300000, 100], ["线下", 4000, 200000, 100]] + result = await analyze("differ", "mean", snaps=[snap("Q1", sql, ["channel", "s", "ss", "n"], rows)], query="Q1", + by="channel", sum="s", sumsq="ss", n="n", label="客单价") + va = (300000 - 5000 ** 2 / 100) / 99 + vb = (200000 - 4000 ** 2 / 100) / 99 + expected = stats.welch(50, va, 100, 40, vb, 100) + assert Fraction(result["values"]["p"]) == pytest.approx(expected.p, rel=1e-9) + assert result["verdict"]["code"] == "differs" and result["values"]["test"] == "welch" + assert result["phrase"] == "线上的客单价（50）高于线下（40），差异站得住" + assert result["method"] == "differ.welch@1" and result["settings"] == {"differ_alpha": 0.05, "min_n": 30} + + +async def test_means_from_mean_sd_and_n_and_small_samples(): + sql = "SELECT channel, avg_amount, sd_amount, n FROM channel_stats" + rows = [["线上", 52, 12, 12], ["线下", 50, 11, 400]] + result = await analyze("differ", "mean", snaps=[snap("Q1", sql, ["channel", "avg_amount", "sd_amount", "n"], rows)], + query="Q1", by="channel", mean="avg_amount", sd="sd_amount", n="n", label="客单价") + assert result["verdict"]["code"] == "no_evidence" + note = next(c for c in result["caveats"] if c["code"] == "small_sample") + assert note["text"] == "有一组的样本数不到 30（线上 12），结论不稳" + assert result["phrase"].startswith("线上和线下的客单价差异可能只是样本波动（样本量 12、400）") + + +async def test_rows_mode_flags_a_single_value_that_flips_the_verdict(): + base = [["线下", v] for v in [40, 60] * 200] + online = [["线上", f"{v:.1f}"] for v in [41.3, 61.3] * 200] + online[-1] = ["线上", "231.3"] + s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], online + base) + result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount", label="客单价") + assert result["method"] == "differ.welch_rows@1" + assert result["verdict"]["code"] == "differs" + note = next(c for c in result["caveats"] if c["code"] == "outlier_sensitive") + assert note["text"] == "去掉最大的一笔后，差异不再站得住" + assert note["text"] in result["phrase"] + hidden = [["线上", 50 + (i % 7)] for i in range(30)] + [["线下", 48 + (i % 5)] for i in range(30)] + [["线上", 400]] + s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], hidden) + flipped = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount", label="客单价") + assert flipped["verdict"]["code"] == "no_evidence" + assert any(c["text"] == "去掉最大的一笔后，差异变得站得住" for c in flipped["caveats"]) + + +async def test_two_periods_of_the_same_group(): + sql = "SELECT week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders WHERE region = '西南' GROUP BY week" + s = snap("Q1", sql, ["week", "refunds", "orders"], [["W41", 30, 1000], ["W42", 60, 1000], ["W40", 31, 1000]]) + result = await analyze("differ", "rate", snaps=[s], query="Q1", period="week", base="W41", target="W42", + numerator="refunds", denominator="orders", label="退款率") + assert result["verdict"] == {"code": "differs", "subjects": [["W42"], ["W41"]], "direction": "up"} + assert result["inputs"][0]["rows"] == [0, 1] + assert result["phrase"] == "W42 的退款率（6.0%）高于 W41（3.0%），差异站得住" + + +async def test_a_group_with_one_value_cannot_be_tested(): + s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], [["线上", 5], ["线下", 4], ["线下", 6]]) + result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount") + assert result["verdict"]["code"] == "no_evidence" and result["values"]["p"] is None + assert result["table"]["rows"][0][-1] == "样本不够，算不了" + assert math.isfinite(float(result["values"]["diff"])) diff --git a/backend/tests/test_analysis_probe.py b/backend/tests/test_analysis_probe.py new file mode 100644 index 00000000..f84a873e --- /dev/null +++ b/backend/tests/test_analysis_probe.py @@ -0,0 +1,129 @@ +"""可比性检查的现查（app/analysis/probe.py）：make_prober 在真的数据源上探表的日期范围、在库里算按运行当天现算的界，遮罩列 +不探、超时限不探、现查不成退回剖析事实；Book 先登记、查完再算，结论只记最后真用到的。库是现造的 SQLite 文件，表名、列名、 +数字都是虚构的。""" +from __future__ import annotations + +import sqlite3 +import uuid +from datetime import date, datetime, timedelta, timezone + +import pytest + +from analysis_samples import CATALOGS, recomputed, snap +from app import analysis +from app.analysis.contract import effective_settings, same +from app.analysis.probe import Book, make_prober +from app.data import introspect +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource + +CLOCK = "2026-10-21T09:30:00" +#: SQLite 的 date('now') 按 UTC 算：表里的数据造到 UTC 的昨天 +UTC_TODAY = datetime.now(timezone.utc).date() +LAST_DAY = UTC_TODAY - timedelta(days=1) + + +@pytest.fixture +async def shop(tmp_path): + """orders：order_date 从 2026-01-01 到昨天（UTC），每天一单；customers 的 signup_date、phone 是遮罩列。""" + path = tmp_path / f"{uuid.uuid4().hex[:8]}.db" + db = sqlite3.connect(path) + db.executescript("CREATE TABLE orders (order_date TEXT, region TEXT, amount INTEGER);" + "CREATE TABLE customers (signup_date TEXT, phone TEXT);" + "INSERT INTO customers VALUES ('2026-01-05', '13900000000');") + day = date(2026, 1, 1) + while day <= LAST_DAY: + db.execute("INSERT INTO orders VALUES (?, '华东', 100)", (day.isoformat(),)) + day += timedelta(days=1) + db.commit() + db.close() + name = f"shop_{uuid.uuid4().hex[:8]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True, + options={"mask_columns": ["phone", "signup_date"]}) + session.add(row) + await session.flush() + row.schema_cache = await introspect.introspect(row) + await session.commit() + await session.refresh(row) + yield row + await engines.invalidate(row.id) + + +async def test_the_live_range_of_a_column(shop): + prober = make_prober({shop.name: shop}, clock=lambda: CLOCK) + probe = await prober(shop.name, "orders", "order_date") + assert probe == {"source": shop.name, "table": "orders", "column": "order_date", "min": "2026-01-01", + "max": LAST_DAY.isoformat(), "basis": "live", "at": CLOCK} + assert prober.dialect(shop.name) == "sqlite" and prober.dialect("nope") is None + assert await prober("nope", "orders", "order_date") is None + + +async def test_masked_columns_are_never_probed(shop): + prober = make_prober({shop.name: shop}) + assert await prober(shop.name, "customers", "signup_date") is None # 数据源设的遮罩列 + explicit = make_prober({shop.name: shop}, masked={shop.name: ["ORDER_DATE"]}) + assert await explicit(shop.name, "orders", "order_date") is None + + +async def test_a_failed_probe_falls_back_to_profile_facts(shop): + async def facts(table): + return {"at": "2026-10-20T02:00:00", "date_ranges": [{"column": "paid_on", "min": "2026-02-01", + "max": "2026-10-19"}]} + + prober = make_prober({shop.name: shop}, facts={shop.name: facts}, clock=lambda: CLOCK) + probe = await prober(shop.name, "payments", "paid_on") # 没有这张表：现查不成 + assert probe == {"source": shop.name, "table": "payments", "column": "paid_on", "min": "2026-02-01", + "max": "2026-10-19", "basis": "profile", "at": "2026-10-20"} + + +async def test_the_budget_stops_further_probes(shop): + prober = make_prober({shop.name: shop}, budget=0) + await prober(shop.name, "orders", "order_date") + assert await prober(shop.name, "orders", "order_date") is None + + +async def test_bounds_are_computed_in_the_database(shop): + prober = make_prober({shop.name: shop}, clock=lambda: CLOCK) + probe = await prober.bounds(shop.name, "DATE('now', '-7 days')") + assert probe["expr"] == "DATE('now', '-7 days')" and probe["basis"] == "live" and probe["at"] == CLOCK + assert date.fromisoformat(probe["value"]) == UTC_TODAY - timedelta(days=7) + assert await prober.bounds(shop.name, "no_such_function(1)") is None + + +def test_the_book_registers_first_and_records_only_what_was_used(): + book = Book({}) + assert book.range("shop", "orders", "order_date") is None and book.bound("shop", "DATE('now')") is None + assert book.missing == [("range", "shop", "orders", "order_date"), ("bound", "shop", "DATE('now')")] + found = {"source": "shop", "table": "orders", "column": "order_date", "min": "2026-01-01", "max": "2026-10-18", + "basis": "live", "at": CLOCK} + filled = Book({("range", "shop", "orders", "order_date"): found, ("bound", "shop", "DATE('now')"): None}) + assert filled.range("shop", "ORDERS", "Order_Date") is found # 表名、列名不分大小写 + filled.bound("shop", "DATE('now')") + assert filled.missing == [] and filled.used == [found] + replay = Book([found]) # 复算：记下的就是全部 + assert replay.bound("shop", "DATE('now', '-1 day')") is None and replay.missing == [] + + +async def test_a_full_comparability_check_against_the_database_records_its_probes(shop): + sql = ("SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', '{a}') " + "AND order_date < date('now', '{b}')") + base = snap("Q1", sql.format(a="-14 days", b="-7 days"), ["gmv"], [[700]], source=shop.name) + target = snap("Q2", sql.format(a="-7 days", b="0 days"), ["gmv"], [[300]], source=shop.name) + prober = make_prober({shop.name: shop}, clock=lambda: CLOCK) + result = await analysis.run({"op": "comparable", "params": {"base_query": "Q1", "target_query": "Q2", + "measure": "gmv"}}, + {"Q1": base, "Q2": target}.get, settings=effective_settings(None), + context={"today": UTC_TODAY, "probe": prober, + "catalogs": {shop.name: CATALOGS["shop"]}}) + exprs = [p["expr"] for p in result["probes"] if "expr" in p] + assert exprs == ["DATE('now', '-14 days')", "DATE('now', '-7 days')", "DATE('now', '0 days')"] + ranges = [p for p in result["probes"] if "table" in p] + assert [(p["table"], p["column"], p["max"], p["basis"]) for p in ranges] == \ + [("orders", "order_date", LAST_DAY.isoformat(), "live")] + assert result["values"]["base_days"] == "7" and result["values"]["target_days"] == "7" + assert result["values"]["target_end"] == LAST_DAY.isoformat() + assert all(i["dialect"] == "sqlite" for i in result["inputs"]) + assert result["verdict"]["code"] == "comparable" and result["phrase"] == "两期可比" + assert same(result, recomputed(result, [base, target])) diff --git a/backend/tests/test_analysis_run.py b/backend/tests/test_analysis_run.py new file mode 100644 index 00000000..262716ca --- /dev/null +++ b/backend/tests/test_analysis_run.py @@ -0,0 +1,152 @@ +"""analysis.run / recompute（app/analysis/__init__.py）：结论的形状严格按契约、同输入两次 run 逐字节相同、复算和运行逐字段相同且 +只用记下的 probes（不现查、不读目录）、调用方写的系统字段不收、复算取不回输入时的说法。""" +from __future__ import annotations + +import json +from datetime import date, timedelta + +import pytest + +from analysis_samples import CATALOGS, analyze, call_error, fixed_prober, recomputed, snap +from app import analysis +from app.analysis import contract as c +from app.analysis.contract import CallError, effective_settings, same + +WEEKLY_SQL = "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week" +WEEKLY = snap("Q2", WEEKLY_SQL, ["region", "week", "gmv"], + [["华东", "2026-W41", 820], ["华东", "2026-W42", 700], ["华北", "2026-W41", 600], + ["华北", "2026-W42", 560], ["华南", "2026-W41", 700], ["华南", "2026-W42", 666]]) +SERIES = snap("Q1", "SELECT DATE(order_date, 'weekday 0', '-6 days') AS week, SUM(amount) AS gmv FROM orders GROUP BY 1", + ["week", "gmv"], [[(date(2026, 6, 1) + timedelta(days=7 * i)).isoformat(), 100 + (i % 4) * 3 + i] + for i in range(20)]) +RATES = snap("Q3", "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region", + ["region", "refunds", "orders"], [["西南", 80, 1000], ["华东", 40, 1000], ["华北", 41, 1000]]) +BASE = snap("Q4", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' AND " + "order_date < '2026-10-12' GROUP BY region", ["region", "gmv"], [["华东", 820], ["华北", 600]]) +TARGET = snap("Q5", "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-12' AND " + "order_date < '2026-10-19' GROUP BY region", ["region", "gmv"], [["华东", 700], ["华北", 590]]) +SNAPS = [WEEKLY, SERIES, RATES, BASE, TARGET] +PROBE = {("orders", "order_date"): ("2025-01-01", "2026-10-20")} + +CALLS = [ + ("decompose", "additive", {"query": "Q2", "by": "region", "period": "week", "base": "2026-W41", + "target": "2026-W42", "measure": "gmv", "label": "销售额"}), + ("decompose", "additive", {"base_query": "Q4", "target_query": "Q5", "by": "region", "measure": "gmv"}), + ("baseline", None, {"query": "Q1", "period": "week", "measure": "gmv"}), + ("trend", None, {"query": "Q1", "period": "week", "measure": "gmv"}), + ("differ", "rate", {"query": "Q3", "by": "region", "numerator": "refunds", "denominator": "orders"}), + ("comparable", None, {"base_query": "Q4", "target_query": "Q5", "measure": "gmv"}), +] + + +async def _run(op, mode, params, *, probe=None, **kw): + return await analyze(op, mode, snaps=SNAPS, probe=probe or fixed_prober(PROBE), check=False, **params, **kw) + + +def _assert_shape(result): + """结论的字段、取值严格按契约。""" + assert set(result) == set(c.AnalysisResult.__annotations__) + assert result["op"] in c.OPS and result["method"] == c.METHODS[(result["op"], result["mode"])] + assert result["status"] in ("ok", "refused") + if result["status"] == "refused": + assert result["reason_code"] in c.REASONS and result["reason"] and result["verdict"] is None + else: + assert result["reason_code"] is None and result["reason"] is None + assert result["verdict"]["code"] in c.VERDICTS[result["op"]] + assert set(result["verdict"]) == {"code", "subjects", "direction"} + assert all(isinstance(s, list) and all(isinstance(x, str) for x in s) for s in result["verdict"]["subjects"]) + assert result["verdict"]["direction"] in (None, "up", "down") + table = result["table"] + assert set(table) == {"columns", "labels", "rows"} and set(table["labels"]) == set(table["columns"]) + assert all(len(r) == len(table["columns"]) for r in table["rows"]) + assert all(v is None or isinstance(v, str) for v in result["values"].values()) + assert all(set(cv) == {"code", "text"} and cv["code"] in c.CAVEATS for cv in result["caveats"]) + assert set(result["settings"]) <= set(c.DEFAULTS) + assert all(i.get("artifact") for i in result["inputs"]) + assert result["criterion"] and result["phrase"] + json.dumps(result, ensure_ascii=False) # 写得进 JSON + + +@pytest.mark.parametrize("op,mode,params", CALLS) +async def test_every_result_has_the_contract_shape_and_recomputes_the_same(op, mode, params): + result = await _run(op, mode, params) + _assert_shape(result) + again = recomputed(result, SNAPS) + assert same(result, again) + assert json.dumps(again, sort_keys=True, ensure_ascii=False) == json.dumps(result, sort_keys=True, ensure_ascii=False) + + +@pytest.mark.parametrize("op,mode,params", CALLS) +async def test_the_same_inputs_give_byte_identical_results(op, mode, params): + one = await _run(op, mode, params) + two = await _run(op, mode, params) + assert json.dumps(one, sort_keys=True, ensure_ascii=False) == json.dumps(two, sort_keys=True, ensure_ascii=False) + + +async def test_recompute_uses_the_recorded_probes_and_facts_not_live_ones(): + result = await _run("comparable", None, {"base_query": "Q4", "target_query": "Q5", "measure": "gmv"}) + assert result["verdict"]["code"] == "comparable" and len(result["probes"]) == 1 + # 记下的范围换一个（表里数据只到 10-16）：复算按记下的算，结论跟着变——它没有现查 + edited = {**result, "probes": [{**result["probes"][0], "max": "2026-10-16"}]} + assert recomputed(edited, SNAPS)["verdict"]["code"] == "incomparable" + # 目录事后改了也不影响复算：目录事实记在 params 里 + decompose = await _run(*CALLS[0]) + assert decompose["params"]["catalog"]["labels"]["region"] == "地区" + CATALOGS["shop"]["orders"]["columns"]["region"]["label"]["value"] = "大区" + try: + assert same(decompose, recomputed(decompose, SNAPS)) + finally: + CATALOGS["shop"]["orders"]["columns"]["region"]["label"]["value"] = "地区" + + +async def test_system_params_written_by_the_caller_are_ignored(): + params = {"query": "Q1", "period": "week", "measure": "gmv", "today": "2020-01-01", + "catalog": {"labels": {"gmv": "假的叫法"}}, "depth": 9} + result = await analysis.run({"op": "baseline", "params": params}, {s["ref"]: s for s in SNAPS}.get, + settings=effective_settings(None), + context={"today": date(2026, 10, 21), "catalogs": CATALOGS}) + assert result["params"]["today"] == "2026-10-21" + assert "假的叫法" not in result["phrase"] and "depth" not in result["params"] + + +async def test_analysis_card_overrides_reach_the_criterion_and_settings(): + params = CALLS[0][2] + result = await _run("decompose", "additive", params, settings={"main_share": 0.6}) + assert result["settings"]["main_share"] == 0.6 and "占总变化的 60% 以上" in result["criterion"] + + +async def test_call_errors_for_unknown_ops_and_modes(): + with pytest.raises(CallError) as e: + await analysis.run({"op": "forecast", "params": {}}, {}.get, settings=effective_settings(None), context={}) + assert e.value.code == "unknown_op" and "变化拆解" in e.value.message + with pytest.raises(CallError) as e: + await analysis.run({"op": "decompose", "mode": "share", "params": {}}, {}.get, + settings=effective_settings(None), context={}) + assert e.value.code == "unknown_mode" and "additive / ratio / factor" in e.value.message + missing = await call_error("baseline", snaps=SNAPS, query="Q1", period="week") + assert missing.code == "missing_param" and missing.message == "没写度量列" + + +async def test_recompute_says_when_an_input_cannot_be_loaded(): + result = await _run(*CALLS[0]) + with pytest.raises(CallError) as e: + analysis.recompute(result, lambda artifact: None) + assert e.value.code == "input_missing" and e.value.message == "复算取不回 Q2 的查询快照" + + +async def test_the_hypothesis_is_recorded_but_never_rendered(): + result = await _run(*CALLS[0], hypothesis="可能是华东促销结束 30% 的影响") + assert result["hypothesis"] == "可能是华东促销结束 30% 的影响" + assert "促销" not in result["phrase"] and "30" not in result["phrase"] + assert same(result, {**recomputed(result, SNAPS), "hypothesis": None}) + + +async def test_ledger_entries_name_the_real_input_snapshots(): + result = await _run(*CALLS[1]) + entry = c.ledger_entry(result, node_id="agent", exec_no=1, artifact="f" * 64, call_id="call-1") + assert entry["inputs"] == [BASE["artifact"], TARGET["artifact"]] and entry["op"] == "decompose" + + +def test_the_call_limit_error_is_spoken_with_the_limit(): + error = analysis.exhausted({"max_calls": 5}) + assert error.code == "max_calls" and error.message == "这次运行的分析次数已经用完（最多 5 次）：用已有的分析结论写，不再新做分析" diff --git a/backend/tests/test_analysis_series.py b/backend/tests/test_analysis_series.py new file mode 100644 index 00000000..bb4c4774 --- /dev/null +++ b/backend/tests/test_analysis_series.py @@ -0,0 +1,210 @@ +"""波动基线、趋势（app/analysis/series.py，细案 01 §3、§4）：常规 / 偏离 / 明显偏离、σ̂ = 0、历史不够、本期不完整、缺期、日数据 +lag 7、新高新低；上升 / 下降 / 无趋势、连续期数、缺期。每个用例都顺带核对复算和运行逐字段相同。""" +from __future__ import annotations + +from datetime import date, timedelta +from fractions import Fraction + +import pytest + +from analysis_samples import analyze, call_error, fixed_prober, snap +from app.analysis.series import MAD_SCALE, run_length + +SQL = "SELECT DATE(order_date, 'weekday 0', '-6 days') AS week, COUNT(*) AS orders FROM orders GROUP BY 1" +#: 每周一的日期：最后一周是 2026-10-12（到 10-18，运行当天 10-21 之前已经过完） +LAST_MONDAY = date(2026, 10, 12) + + +def weeks(values, *, last=LAST_MONDAY): + start = last - timedelta(days=7 * (len(values) - 1)) + return [[(start + timedelta(days=7 * i)).isoformat(), v] for i, v in enumerate(values)] + + +def series(rows, sql=SQL, cols=("week", "orders")): + return snap("Q1", sql, list(cols), rows, types={cols[0]: "text", cols[1]: "number"}) + + +async def baseline(rows, **params): + params = {"query": "Q1", "period": "week", "measure": "orders", "label": "订单数", **params} + return await analyze("baseline", snaps=[series(rows)], **params) + + +async def trend(rows, **params): + params = {"query": "Q1", "period": "week", "measure": "orders", "label": "线上销售额", **params} + return await analyze("trend", snaps=[series(rows)], **params) + + +NOISE = [100, 102, 98, 101, 99, 103, 100, 97, 102, 101, 99, 100, 98, 103, 101, 100, 99, 102, 100] + + +# -------------------------------------------------------------------------- +# 波动基线 +# -------------------------------------------------------------------------- + + +async def test_level_basis_with_hand_computed_median_and_mad(): + rows = [[i + 1, v] for i, v in enumerate([10, 12, 11, 13, 9, 10, 12, 11, 16])] + result = await baseline(rows, basis="level") + # 历史中位数 11，MAD 1，σ̂ = 1.4826，z = (16 − 11) / 1.4826 + values = result["values"] + assert values["median"] == "11" and values["n_history"] == "8" and values["rank"] == "1" + assert Fraction(values["z"]) == pytest.approx(Fraction(5) / MAD_SCALE) + assert values["band_lo"] == "8.0348" and values["band_hi"] == "13.9652" + assert result["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "up"} + assert result["phrase"].startswith("订单数本期为 16，明显超出过去 8 期的常规波动范围（8.03 至 13.97）") + assert any(c["code"] == "record_high" for c in result["caveats"]) + assert result["params"]["grain"] == "seq" and result["params"]["lag"] == 1 + assert result["settings"] == {"k_warn": 2.0, "k_alert": 3.0, "min_history": 8, "max_history": 52} + assert result["table"]["rows"][-1][-1] == "本期" + + +async def test_change_basis_within_outside_and_far_outside(): + probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")}) + within = await analyze("baseline", snaps=[series(weeks(NOISE + [101]))], probe=probe, query="Q1", period="week", + measure="orders", label="订单数") + assert within["verdict"]["code"] == "within_band" + assert within["phrase"].startswith("订单数本期变化 +1.0%，在过去 18 周的常规波动范围内（") + assert within["params"]["today"] == "2026-10-21" and within["params"]["grain"] == "week" + assert within["probes"][0]["table"] == "orders" and within["probes"][0]["column"] == "order_date" + assert not any(c["code"] == "comparability" for c in within["caveats"]) + outside = await analyze("baseline", snaps=[series(weeks(NOISE + [107]))], probe=probe, query="Q1", + period="week", measure="orders") + assert outside["verdict"]["code"] == "outside_band" and "超出过去 18 周" in outside["phrase"] + far = await analyze("baseline", snaps=[series(weeks(NOISE + [70]))], probe=probe, query="Q1", period="week", + measure="orders", label="订单数") + assert far["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "down"} + assert far["phrase"].startswith("订单数本期变化 -30.0%，明显超出过去 18 周的常规波动范围") + assert any(c["code"] == "record_low" and c["text"] == "本期的订单数是过去 18 周里最低的" for c in far["caveats"]) + + +async def test_without_a_table_range_completeness_is_said_to_be_unverified_but_not_in_the_phrase(): + result = await baseline(weeks(NOISE + [101])) + note = next(c for c in result["caveats"] if c["code"] == "comparability") + assert note["text"] == "没能核对本期的数据是否到齐（查不到表里这一列的日期范围）" + assert note["text"] not in result["phrase"] + + +async def test_flat_history_has_no_spread(): + rows = [[i + 1, 50] for i in range(10)] + same = await baseline(rows, basis="level") + assert same["verdict"]["code"] == "within_band" and same["values"]["z"] is None + assert same["phrase"] == "订单数本期为 50，和过去 9 期一样（每期都是 50）" + jump = await baseline(rows[:-1] + [[10, 51]], basis="level") + assert jump["verdict"] == {"code": "far_outside_band", "subjects": [], "direction": "up"} + assert jump["phrase"].startswith("订单数本期为 51，明显超出过去 9 期的常规波动范围（每期都是 50）") + + +async def test_short_history_partial_target_gaps_and_zero_bases_are_refused(): + short = await baseline(weeks([100, 101, 99, 102, 98, 100])) + # 6 周：第一周没有上一周可比，本期不算历史，历史的变化只有 4 个 + assert short["reason_code"] == "history_short" and short["reason"] == "只有 4 期历史，至少要 8 期" + assert short["phrase"] == "波动基线没有做成：只有 4 期历史，至少要 8 期" + # 最后一周是 10-19 到 10-25：运行当天 10-21 还没过完 + partial = await baseline(weeks(NOISE + [60], last=date(2026, 10, 19))) + assert partial["reason_code"] == "partial_target" + assert partial["reason"] == "本期数据只到 2026-10-20，还差 5 天，和完整的周比没有意义" + # 周过完了，但表里的数据只到 10-17 + probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-17")}) + late = await analyze("baseline", snaps=[series(weeks(NOISE + [60]))], probe=probe, query="Q1", period="week", + measure="orders") + assert late["reason"] == "本期数据只到 2026-10-17，还差 1 天，和完整的周比没有意义" + gap_rows = weeks(NOISE + [100, 100]) + del gap_rows[-2] # 本期的上一周没有 + gap = await baseline(gap_rows) + assert gap["reason_code"] == "target_gap" + zero = await baseline(weeks(NOISE + [0, 10])) + assert zero["reason_code"] == "zero_base" + unreadable = await baseline([["上周", 1], ["本周", 2]]) + assert unreadable["reason_code"] == "period_unreadable" and "上周" in unreadable["reason"] + + +async def test_daily_series_compare_with_the_same_weekday_last_week(): + start = date(2026, 7, 1) + rows = [[(start + timedelta(days=i)).isoformat(), 100 + (i % 7) * 10] for i in range(100)] + result = await analyze("baseline", snaps=[series(rows, cols=("day", "orders"))], query="Q1", period="day", + measure="orders", label="订单数", check=True) + assert result["params"]["grain"] == "day" and result["params"]["lag"] == 7 + # 星期效应：和上周同一天比，每天的变化都是 0 + assert result["values"]["median"] == "0" and result["verdict"]["code"] == "within_band" + assert result["phrase"].startswith("订单数比上周同一天变化 0.0%，和过去 52 天一样") + yoy = await baseline(weeks(NOISE * 3), lag=52) # 57 周：能和 52 周前比的历史只有 4 期 + assert yoy["reason_code"] == "history_short" and yoy["reason"] == "只有 4 期历史，至少要 8 期" + assert (await baseline(weeks(NOISE * 4), lag=52))["status"] == "ok" + assert (await call_error("baseline", snaps=[series(weeks(NOISE))], query="Q1", period="week", measure="orders", + lag=0)).code == "bad_lag" + + +async def test_duplicate_periods_and_given_targets(): + rows = weeks(NOISE + [101]) + error = await call_error("baseline", snaps=[series(rows + [rows[-1]])], query="Q1", period="week", + measure="orders") + assert error.code == "duplicate_rows" and "在 SQL 里按 week 汇总" in error.message + earlier = await baseline(rows, target=rows[-3][0]) + assert earlier["params"]["target"] == rows[-3][0] and earlier["values"]["n_history"] == "16" + missing = await call_error("baseline", snaps=[series(rows)], query="Q1", period="week", measure="orders", + target="2020-01-06") + assert missing.code == "period_not_found" + + +async def test_iso_week_labels_and_months(): + labels = [[f"2026-W{w:02d}", v] for w, v in zip(range(22, 42), NOISE + [101])] + result = await baseline(labels) + assert result["params"]["grain"] == "week" and result["verdict"]["code"] == "within_band" + months = [[f"2025-{m:02d}", 100 + m] for m in range(1, 13)] + [[f"2026-{m:02d}", 112 + m] for m in range(1, 10)] + monthly = await baseline(months) + assert monthly["params"]["grain"] == "month" and "过去 19 个月" in monthly["phrase"] + + +# -------------------------------------------------------------------------- +# 趋势 +# -------------------------------------------------------------------------- + + +async def test_rising_and_falling_trends_with_run_length(): + rising = await trend(weeks([100 + 3 * i + (i % 3) for i in range(12)])) + assert rising["verdict"] == {"code": "rising", "subjects": [], "direction": "up"} + assert rising["values"]["n"] == "12" and rising["values"]["run_length"] == "11" + assert rising["phrase"].startswith("过去 12 周线上销售额呈上升趋势（每周约 +") + assert "，已连续 11 周上升" in rising["phrase"] + assert rising["settings"] == {"trend_min_points": 6, "trend_alpha": 0.05, "max_history": 52} + falling = await trend(weeks([200, 196, 197, 190, 185, 186, 180, 181, 175, 170])) + assert falling["verdict"]["code"] == "falling" + assert falling["values"]["run_length"] == "2" and "已连续" not in falling["phrase"] + + +async def test_no_trend_and_exact_small_sample_p(): + flat = await trend(weeks([100, 104, 98, 103, 97, 101, 99, 102])) + assert flat["verdict"] == {"code": "no_trend", "subjects": [], "direction": None} + assert flat["phrase"] == "过去 8 周线上销售额没有明显的上升或下降趋势" + six = await trend(weeks([1, 2, 3, 4, 5, 6])) + # 没有并列、n ≤ 10：精确分布，单调的 6 期双侧 p = 2 / 6! + assert Fraction(six["values"]["p"]) == pytest.approx(Fraction(2, 720)) + assert six["verdict"]["code"] == "rising" and six["values"]["slope"] == "1" + + +def test_run_length_counts_consecutive_changes_from_the_end(): + assert run_length([Fraction(v) for v in (1, 2, 3, 2, 3, 4, 5)]) == (3, "up") + assert run_length([Fraction(v) for v in (5, 4, 3)]) == (2, "down") + assert run_length([Fraction(v) for v in (1, 2, 2)]) == (0, None) + + +async def test_trend_needs_enough_consecutive_points(): + rows = weeks([100 + i for i in range(10)]) + del rows[-4] # 最后连着的只有 3 期 + gap = await trend(rows) + assert gap["reason_code"] == "series_gap" and gap["reason"] == "最后连着的只有 3 期，再往前缺期（至少要连着的 6 期）" + short = await trend(weeks([1, 2, 3, 4])) + assert short["reason_code"] == "history_short" and short["reason"] == "只有 4 期，至少要 6 期" + partial = await trend(weeks([100 + i for i in range(10)], last=date(2026, 10, 19))) + assert partial["reason_code"] == "partial_target" + + +async def test_daily_trends_warn_about_weekday_effects(): + start = date(2026, 8, 1) + rows = [[(start + timedelta(days=i)).isoformat(), 100 + i] for i in range(30)] + result = await analyze("trend", snaps=[series(rows, cols=("day", "orders"))], query="Q1", period="day", + measure="orders") + assert result["verdict"]["code"] == "rising" + assert any(c["code"] == "weekday_effect" and c["text"] == "日数据有星期效应，看趋势建议按周汇总" + for c in result["caveats"]) + assert "日数据有星期效应" in result["phrase"] From ab98c7430cbccbf13bd081c73daa94df260e9628 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 13:08:14 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E5=88=86=E6=9E=90=E7=AE=97=E5=AD=90?= =?UTF-8?q?=E6=94=B6=E5=B0=BE=EF=BC=9A=E5=A4=8D=E7=AE=97=E5=89=94=E9=99=A4?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F=E5=AD=97=E6=AE=B5=E6=94=B9=E7=94=A8=E5=90=8C?= =?UTF-8?q?=E4=B8=80=E4=B8=AA=20DERIVED=5FPARAMS=EF=BC=9B=E5=8E=BB?= =?UTF-8?q?=E6=8E=89=E6=B2=A1=E7=94=A8=E7=9A=84=E5=AF=BC=E5=85=A5=E5=92=8C?= =?UTF-8?q?=E5=B0=8F=E5=87=BD=E6=95=B0=EF=BC=9B=E6=B5=8B=E8=AF=95=E6=8C=89?= =?UTF-8?q?=E4=BB=93=E5=BA=93=E7=9A=84=E5=86=99=E6=B3=95=E4=BB=8E=20tests.?= =?UTF-8?q?analysis=5Fsamples=20=E5=AF=BC=E5=85=A5=E6=A0=B7=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/__init__.py | 3 ++- backend/app/analysis/differ.py | 1 - backend/app/analysis/series.py | 4 ---- backend/tests/test_analysis_comparable.py | 2 +- backend/tests/test_analysis_decompose.py | 2 +- backend/tests/test_analysis_differ.py | 2 +- backend/tests/test_analysis_probe.py | 2 +- backend/tests/test_analysis_run.py | 2 +- backend/tests/test_analysis_series.py | 2 +- 9 files changed, 8 insertions(+), 12 deletions(-) diff --git a/backend/app/analysis/__init__.py b/backend/app/analysis/__init__.py index 67c4ec37..895df813 100644 --- a/backend/app/analysis/__init__.py +++ b/backend/app/analysis/__init__.py @@ -111,6 +111,7 @@ def recompute(result: AnalysisResult, load: Callable[[str], InputSnapshot | None """复算：load 按 sha256 取回输入快照，load_result 按 sha256 取回 parent 那份结论。用结论里记下的 params（含运行当天、 目录事实）、settings、probes，不现查、不读目录。取不回输入时抛 CallError（code input_missing）；方法升版后调用不再成立 时同样抛 CallError（照 run 的说法）。和封存的那份用 contract.same 比。""" + from app.analysis.common import DERIVED_PARAMS from app.analysis.facts import Facts from app.analysis.inputs import _CELL, Inputs from app.analysis.probe import Book @@ -149,7 +150,7 @@ def recompute(result: AnalysisResult, load: Callable[[str], InputSnapshot | None today = date.fromisoformat(params["today"]) if isinstance(params.get("today"), str) else None env = _env(op, mode, settings, Inputs(resolve, dialect), Facts(recorded=params.get("catalog") or {}), Book(result.get("probes") or []), today, parents, result.get("hypothesis")) - raw = {k: v for k, v in params.items() if k not in ("catalog", "today", "depth")} + raw = {k: v for k, v in params.items() if k not in DERIVED_PARAMS} return _compute(op, mode, raw, env) diff --git a/backend/app/analysis/differ.py b/backend/app/analysis/differ.py index a6d5e7ef..733ceb52 100644 --- a/backend/app/analysis/differ.py +++ b/backend/app/analysis/differ.py @@ -21,7 +21,6 @@ direction 是 A 相对 B。 """ from __future__ import annotations -import math from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from fractions import Fraction diff --git a/backend/app/analysis/series.py b/backend/app/analysis/series.py index 8bf7267a..37f022a0 100644 --- a/backend/app/analysis/series.py +++ b/backend/app/analysis/series.py @@ -111,10 +111,6 @@ def _completeness(env: Env, series: Series, item: pd.Period) -> tuple[str, str | return cmp.span_coverage(span, env.today, env.book) # type: ignore[arg-type] -def _shown_period(item: pd.Period) -> Any: - return item.raw - - # -------------------------------------------------------------------------- # 波动基线 # -------------------------------------------------------------------------- diff --git a/backend/tests/test_analysis_comparable.py b/backend/tests/test_analysis_comparable.py index bf9bb3d8..5b9c514d 100644 --- a/backend/tests/test_analysis_comparable.py +++ b/backend/tests/test_analysis_comparable.py @@ -3,8 +3,8 @@ probes，不现查）。""" from __future__ import annotations -from analysis_samples import analyze, fixed_prober, snap from app.analysis import comparable_reasons +from tests.analysis_samples import analyze, fixed_prober, snap WINDOW = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= '{a}' AND order_date < '{b}'" RANGE = {("orders", "order_date"): ("2025-01-01", "2026-10-20")} diff --git a/backend/tests/test_analysis_decompose.py b/backend/tests/test_analysis_decompose.py index 8e34a692..129c4b76 100644 --- a/backend/tests/test_analysis_decompose.py +++ b/backend/tests/test_analysis_decompose.py @@ -9,8 +9,8 @@ from fractions import Fraction import pytest -from analysis_samples import analyze, call_error, fixed_prober, notes, snap from app.analysis.decompose import lmdi, ratio_effects +from tests.analysis_samples import analyze, call_error, fixed_prober, notes, snap SQL = "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week" TYPES = {"region": "text", "week": "text", "gmv": "number"} diff --git a/backend/tests/test_analysis_differ.py b/backend/tests/test_analysis_differ.py index b8120d7b..5f7e10c0 100644 --- a/backend/tests/test_analysis_differ.py +++ b/backend/tests/test_analysis_differ.py @@ -7,8 +7,8 @@ from fractions import Fraction import pytest -from analysis_samples import analyze, call_error, snap from app.analysis import stats +from tests.analysis_samples import analyze, call_error, snap RATE_SQL = "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region" diff --git a/backend/tests/test_analysis_probe.py b/backend/tests/test_analysis_probe.py index f84a873e..b358fed9 100644 --- a/backend/tests/test_analysis_probe.py +++ b/backend/tests/test_analysis_probe.py @@ -9,7 +9,6 @@ from datetime import date, datetime, timedelta, timezone import pytest -from analysis_samples import CATALOGS, recomputed, snap from app import analysis from app.analysis.contract import effective_settings, same from app.analysis.probe import Book, make_prober @@ -17,6 +16,7 @@ from app.data import introspect from app.data.engine import engines from app.db.base import SessionLocal from app.db.models import DataSource +from tests.analysis_samples import CATALOGS, recomputed, snap CLOCK = "2026-10-21T09:30:00" #: SQLite 的 date('now') 按 UTC 算：表里的数据造到 UTC 的昨天 diff --git a/backend/tests/test_analysis_run.py b/backend/tests/test_analysis_run.py index 262716ca..6baa0ccb 100644 --- a/backend/tests/test_analysis_run.py +++ b/backend/tests/test_analysis_run.py @@ -7,10 +7,10 @@ from datetime import date, timedelta import pytest -from analysis_samples import CATALOGS, analyze, call_error, fixed_prober, recomputed, snap from app import analysis from app.analysis import contract as c from app.analysis.contract import CallError, effective_settings, same +from tests.analysis_samples import CATALOGS, analyze, call_error, fixed_prober, recomputed, snap WEEKLY_SQL = "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week" WEEKLY = snap("Q2", WEEKLY_SQL, ["region", "week", "gmv"], diff --git a/backend/tests/test_analysis_series.py b/backend/tests/test_analysis_series.py index bb4c4774..b09c5fed 100644 --- a/backend/tests/test_analysis_series.py +++ b/backend/tests/test_analysis_series.py @@ -7,8 +7,8 @@ from fractions import Fraction import pytest -from analysis_samples import analyze, call_error, fixed_prober, snap from app.analysis.series import MAD_SCALE, run_length +from tests.analysis_samples import analyze, call_error, fixed_prober, snap SQL = "SELECT DATE(order_date, 'weekday 0', '-6 days') AS week, COUNT(*) AS orders FROM orders GROUP BY 1" #: 每周一的日期：最后一周是 2026-10-12（到 10-18，运行当天 10-21 之前已经过完）