From cf8ec15831985a514e32287df881522cdadf3b41 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 15:35:00 +0800 Subject: [PATCH 01/13] =?UTF-8?q?=E5=91=A8=E5=8F=B7=E6=8C=89=20SQL=20?= =?UTF-8?q?=E9=87=8C=E7=9A=84=E6=A0=BC=E5=BC=8F=E4=B8=B2=E7=AE=97=E7=9C=9F?= =?UTF-8?q?=E5=AE=9E=E8=B5=B7=E6=AD=A2=EF=BC=88=E5=A4=8D=E6=A0=B8=20R3?= =?UTF-8?q?=EF=BC=89=EF=BC=9A=E3=80=8CYYYY-Www=E3=80=8D=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E4=B8=80=E5=BE=8B=E5=BD=93=20ISO=20=E5=91=A8=E3=80=82periods.w?= =?UTF-8?q?eek=5Fscheme=20=E7=9C=8B=E6=9C=9F=E6=AC=A1=E5=88=97=E5=9C=A8?= =?UTF-8?q?=E6=9C=80=E5=A4=96=E5=B1=82=20SELECT=20=E9=87=8C=E7=9A=84?= =?UTF-8?q?=E5=86=99=E6=B3=95=EF=BC=88=E4=BB=8E=20CTE=E3=80=81=E5=AD=90?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E9=87=8C=E5=8E=9F=E6=A0=B7=E5=8F=96=E5=87=BA?= =?UTF-8?q?=E7=9A=84=E5=BE=80=E9=87=8C=E6=89=BE=EF=BC=89=EF=BC=8C=E8=AE=A4?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E5=8C=96=E5=87=BD=E6=95=B0=E9=87=8C=E9=85=8D?= =?UTF-8?q?=E5=A5=97=E7=9A=84=E5=B9=B4=E4=BB=BD=E5=92=8C=E5=91=A8=E5=8F=B7?= =?UTF-8?q?=E2=80=94=E2=80=94SQLite=20strftime=20=E7=9A=84=20%Y-%W=20/=20%?= =?UTF-8?q?Y-%U=20/=20%G-%V=E3=80=81MySQL=20DATE=5FFORMAT=20=E7=9A=84=20%Y?= =?UTF-8?q?-%U=20/=20%Y-%u=20/=20%X-%V=20/=20%x-%v=E3=80=81PostgreSQL=20/?= =?UTF-8?q?=20Oracle=20to=5Fchar=20=E7=9A=84=20YYYY-WW=20/=20IYYY-IW?= =?UTF-8?q?=EF=BC=9Bweek=5Fspan=20=E6=8C=89=E7=AE=97=E6=B3=95=E7=AE=97?= =?UTF-8?q?=E4=B8=80=E5=91=A8=E7=9A=84=E8=B5=B7=E6=AD=A2=EF=BC=88W00?= =?UTF-8?q?=E3=80=81W53=20=E6=8C=89=E5=90=84=E8=87=AA=E7=9A=84=E7=AE=97?= =?UTF-8?q?=E6=B3=95=EF=BC=8C=E4=B8=8D=E8=B5=B0=20fromisocalendar=EF=BC=89?= =?UTF-8?q?=EF=BC=8C=E5=85=88=E5=90=8E=E6=8C=89=E7=AE=97=E6=B3=95=E6=8E=92?= =?UTF-8?q?=EF=BC=88=E8=B7=A8=E5=B9=B4=E5=88=87=E5=BC=80=E7=9A=84=E4=B8=A4?= =?UTF-8?q?=E6=9C=9F=E4=B9=9F=E8=BF=9E=E7=9D=80=EF=BC=89=E3=80=82=E8=AE=A4?= =?UTF-8?q?=E4=B8=8D=E5=87=BA=E7=AE=97=E6=B3=95=EF=BC=88=E6=96=B9=E8=A8=80?= =?UTF-8?q?=E4=B8=8D=E7=9F=A5=E9=81=93=E3=80=81=E5=8F=AA=E5=8F=96=E4=BA=86?= =?UTF-8?q?=E7=8E=B0=E6=88=90=E7=9A=84=E5=AD=97=E7=AC=A6=E4=B8=B2=E3=80=81?= =?UTF-8?q?=E5=B9=B4=E4=BB=BD=E5=92=8C=E5=91=A8=E5=8F=B7=E4=B8=8D=E9=85=8D?= =?UTF-8?q?=E5=A5=97=EF=BC=89=E7=9A=84=E8=B5=B7=E6=AD=A2=E7=BD=AE=E7=A9=BA?= =?UTF-8?q?=EF=BC=8C=E6=9C=AC=E6=9C=9F=E9=BD=90=E4=B8=8D=E9=BD=90=E4=B8=80?= =?UTF-8?q?=E5=BE=8B=E6=A0=B8=E5=AF=B9=E4=B8=8D=E4=BA=86=E3=80=81=E9=99=84?= =?UTF-8?q?=E8=AF=B4=E6=98=8E=EF=BC=8C=E7=BB=9D=E4=B8=8D=E5=88=A4=E9=BD=90?= =?UTF-8?q?=EF=BC=8C=E5=85=88=E5=90=8E=E6=8C=89=E5=91=A8=E5=8F=B7=E6=8E=92?= =?UTF-8?q?=E3=80=82=E6=8C=89=E5=B9=B4=E6=88=AA=E6=96=AD=E7=9A=84=E7=AE=97?= =?UTF-8?q?=E6=B3=95=E9=87=8C=E8=B7=A8=E5=B9=B4=E5=88=87=E5=BC=80=E7=9A=84?= =?UTF-8?q?=E9=82=A3=E4=B8=A4=E6=9C=9F=E4=B8=8D=E5=88=B0=E6=95=B4=E5=91=A8?= =?UTF-8?q?=EF=BC=9A=E5=BD=93=E6=9C=AC=E6=9C=9F=E5=88=A4=20partial=5Ftarge?= =?UTF-8?q?t=EF=BC=8C=E5=BD=93=E6=AF=94=E8=BE=83=E6=9C=9F=E5=88=A4=20targe?= =?UTF-8?q?t=5Fgap=20=E5=B9=B6=E8=AF=B4=E6=B8=85=E5=8F=AA=E6=9C=89?= =?UTF-8?q?=E5=87=A0=E5=A4=A9=EF=BC=8C=E4=B8=8D=E7=AE=97=E8=BF=9B=E6=B3=A2?= =?UTF-8?q?=E5=8A=A8=E5=9F=BA=E7=BA=BF=E7=9A=84=E5=8E=86=E5=8F=B2=EF=BC=88?= =?UTF-8?q?=E9=99=84=20partial=5Fperiods=20=E8=AF=B4=E6=98=8E=EF=BC=8C?= =?UTF-8?q?=E5=8F=AA=E5=9C=A8=E8=AF=81=E6=8D=AE=E9=9D=A2=E6=9D=BF=EF=BC=89?= =?UTF-8?q?=EF=BC=8C=E8=B6=8B=E5=8A=BF=E5=8F=96=E6=9C=80=E5=90=8E=E8=BF=9E?= =?UTF-8?q?=E7=9D=80=E7=9A=84=E6=95=B4=E5=91=A8=E3=80=82=E5=8F=AF=E6=AF=94?= =?UTF-8?q?=E6=80=A7=E6=A3=80=E6=9F=A5=E6=8C=89=E6=9C=9F=E6=AC=A1=E5=88=86?= =?UTF-8?q?=E7=BB=84=E6=97=B6=E4=B9=9F=E8=B5=B0=E8=BF=99=E5=A5=97=EF=BC=9A?= =?UTF-8?q?=E6=9C=89=E4=B8=80=E6=9C=9F=E4=B8=8D=E5=88=B0=E6=95=B4=E5=91=A8?= =?UTF-8?q?=E7=9A=84=E6=8C=89=E5=AE=9E=E9=99=85=E5=A4=A9=E6=95=B0=E6=AF=94?= =?UTF-8?q?=E7=AA=97=E5=8F=A3=E3=80=81=E7=85=A7=E7=AE=97=E6=98=9F=E6=9C=9F?= =?UTF-8?q?=E6=9E=84=E6=88=90=E3=80=82period=5Funreadable=20=E7=9A=84?= =?UTF-8?q?=E5=8E=9F=E5=9B=A0=E9=87=8C=E4=B8=BE=E8=AE=A4=E4=B8=8D=E5=87=BA?= =?UTF-8?q?=E7=9A=84=E9=82=A3=E5=87=A0=E4=B8=AA=E5=80=BC=EF=BC=88=E5=86=99?= =?UTF-8?q?=E6=B3=95=E8=AE=A4=E4=B8=8D=E5=87=BA=E3=80=81=E5=86=99=E6=B3=95?= =?UTF-8?q?=E6=B7=B7=E7=9D=80=E7=9A=84=E5=B0=91=E6=95=B0=E3=80=81=E7=AE=97?= =?UTF-8?q?=E6=B3=95=E9=87=8C=E6=B2=A1=E6=9C=89=E7=9A=84=E9=82=A3=E4=B8=80?= =?UTF-8?q?=E5=91=A8=E3=80=81=E5=AF=B9=E4=B8=8D=E4=B8=8A=E7=BB=99=E7=9A=84?= =?UTF-8?q?=E7=B2=92=E5=BA=A6=E5=88=86=E5=BC=80=E8=AF=B4=EF=BC=89=EF=BC=9B?= =?UTF-8?q?=E6=AF=94=E8=BE=83=E6=9C=9F=E9=82=A3=E4=B8=80=E6=A0=BC=E4=B8=BA?= =?UTF-8?q?=E7=A9=BA=E6=97=B6=E7=85=A7=E5=AE=9E=E8=AF=B4=E4=B8=BA=E7=A9=BA?= =?UTF-8?q?=EF=BC=8C=E4=B8=8D=E5=86=8D=E8=AF=B4=E6=88=90=E7=BC=BA=E6=9C=9F?= =?UTF-8?q?=EF=BC=88=E8=B6=8B=E5=8A=BF=E5=BE=80=E5=89=8D=E6=96=AD=E5=BC=80?= =?UTF-8?q?=E7=9A=84=E5=8E=9F=E5=9B=A0=E5=90=8C=E6=A0=B7=E5=88=86=E5=BC=80?= =?UTF-8?q?=E8=AF=B4=EF=BC=89=E3=80=82=E6=9C=9F=E6=AC=A1=E6=98=AF=E5=BA=8F?= =?UTF-8?q?=E5=8F=B7=E3=80=81=E5=91=A8=E5=8F=B7=E8=AE=A4=E4=B8=8D=E5=87=BA?= =?UTF-8?q?=E7=AE=97=E6=B3=95=E6=97=B6=E4=B9=9F=E9=99=84=E4=B8=80=E5=8F=A5?= =?UTF-8?q?=E6=A0=B8=E5=AF=B9=E4=B8=8D=E4=BA=86=E3=80=82=E5=A5=91=E7=BA=A6?= =?UTF-8?q?=20CAVEATS=20=E5=8A=A0=20partial=5Fperiods?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/comparable.py | 40 ++- backend/app/analysis/contract.py | 2 +- backend/app/analysis/periods.py | 311 ++++++++++++++++++++-- backend/app/analysis/phrases.py | 16 +- backend/app/analysis/series.py | 119 ++++++--- backend/tests/test_analysis_comparable.py | 23 ++ backend/tests/test_analysis_periods.py | 79 +++++- backend/tests/test_analysis_series.py | 103 +++++++ 8 files changed, 624 insertions(+), 69 deletions(-) diff --git a/backend/app/analysis/comparable.py b/backend/app/analysis/comparable.py index 38a6bb9d..3990222e 100644 --- a/backend/app/analysis/comparable.py +++ b/backend/app/analysis/comparable.py @@ -70,7 +70,8 @@ class Span: table: str | None = None column: str | None = None sql_window: Any = None - unit: str | None = None # 这一段是不是整月、整季、整年(calendar_unit),按期次时是粒度 + unit: str | None = None # 这一段是不是整月、整季、整年(calendar_unit),按期次时是粒度(不是整期的为 None) + short: int | None = None # 按期次的这一期不到整期时有几天(periods.short_days:跨年被切开的周) @dataclass @@ -238,7 +239,9 @@ def period_span(label: str, snap: Snap, period: Any, item: pd.Period, grain: str """按期次分组的一份快照里的一期:这一期覆盖的日子,加上 SQL 在期次那一列上的时间条件。""" from app.data import coverage - span = Span(label, snap, item.start, item.end, unit=grain) + span = Span(label, snap, item.start, item.end, unit=grain, short=pd.short_days(item, grain)) + if span.short: + span.unit = None found = date_source(snap.sql, period) if found is None: return span @@ -268,10 +271,13 @@ def _range(span: Span, book: Book) -> tuple[bool, date | None, date | None, bool def span_coverage(span: Span, today: date, book: Book) -> tuple[str, str | None]: - """这一段(窗口或一期)的数据齐不齐:(pass / short / unknown, 不齐的那一句)。没过完(终点不早于运行当天:当天的数据还没 - 到齐)、SQL 的时间条件截掉了一部分、表里的数据没到(最晚早于终点)或来得晚(最早晚于起点),都算不齐。""" + """这一段(窗口或一期)的数据齐不齐:(pass / short / unknown, 不齐的那一句)。这一期本身不到整期(跨年被切开的周)、没过完 + (终点不早于运行当天:当天的数据还没到齐)、SQL 的时间条件截掉了一部分、表里的数据没到(最晚早于终点)或来得晚(最早晚于 + 起点),都算不齐。""" if span.start is None or span.end is None: return "unknown", None + if span.short: + return "short", f"{span.label}只有 {span.start.isoformat()} 到 {span.end.isoformat()} 这 {span.short} 天,不是整周" lo, hi = span.start, span.end window = span.sql_window if window is not None: @@ -417,20 +423,32 @@ def compare_queries(base: Snap, target: Snap, measure: str | None, facts: Facts, def compare_periods(snap: Snap, period: str, base: pd.Period, target: pd.Period, grain: str, measure: str | None, facts: Facts, book: Book, today: date) -> Outcome: - """同一份快照里按期次分组的两期:天然同口径;窗口按期次的粒度;整期看这两期各自齐不齐。""" + """同一份快照里按期次分组的两期:天然同口径;窗口按期次的粒度(有一期不到整期的按实际天数比,星期构成也照算);整期看 + 这两期各自齐不齐。期次不知道是哪几天的(序号、认不出算法的周号)窗口、整期、星期构成都核对不了。""" same = Check("same_query", status="pass") b_span = period_span("基期", snap, period, base, grain, measure, facts, book) t_span = period_span("本期", snap, period, target, grain, measure, facts, book) flow = bool(measure) and facts.measure(snap, measure)[0] == "flow" # type: ignore[arg-type] - if grain == "seq": + whole = not (b_span.short or t_span.short) + if grain == "seq" or base.start is None or target.start is None: checks = [same, Check("window"), Check("complete"), Check("weekdays"), _data_check([("", snap)], measure)] else: - checks = [same, _window_check(b_span, t_span, flow, grouped=True), + checks = [same, _window_check(b_span, t_span, flow, grouped=whole), _complete_check([b_span, t_span], today, book, flow), - _weekday_check(b_span, t_span, grain=grain), _data_check([("", snap)], measure)] + _weekday_check(b_span, t_span, grain=grain if whole else None), _data_check([("", snap)], measure)] return _decide(checks, b_span, t_span) +def period_complete(snap: Snap, period: str, item: pd.Period, grain: str, measure: str | None, facts: Facts, + book: Book, today: date) -> tuple[str, str | None, str | None]: + """按期次分组的一份快照里本期齐不齐(波动基线、趋势、单份快照的变化拆解共用):(pass / short / unknown, 不齐的那一句, + 核对不了时附带说明的那一句)。期次没有日期(序号)、周号认不出算法的,起止不知道,一律核对不了,不判齐。""" + if item.start is None: + return "unknown", None, ph.UNKNOWN_WEEKS if grain == "week" else ph.NO_DATES + status, why = span_coverage(period_span("本期", snap, period, item, grain, measure, facts, book), today, book) + return status, why, ph.UNVERIFIED_COMPLETE if status == "unknown" else None + + def table_of(outcome: Outcome) -> dict[str, Any]: """检查表(结论的 table):检查项、结果、基期、本期、说明。""" rows = [[CHECK_NAMES[c.id], STATUS_NAMES[c.status], c.base, c.target, ";".join(c.reasons) or None] @@ -513,7 +531,7 @@ def comparable(raw: Mapping[str, Any], env: Any) -> Any: raise CallError("period_not_found", f"期次列「{period}」里没有「{params[side]}」:{what}要写 {target.ref} " "里有的期次值") env.inputs.use(target, "data", [period, *([measure] if measure else [])], rows["base"] + rows["target"]) - parsed = pd.parse(values, grain) + parsed = pd.parse(values, grain, weeks=pd.week_scheme(target.sql, period, target.dialect)) if parsed is None: item_b = item_t = pd.Period(None, 0, None, None) grain = "seq" @@ -538,5 +556,5 @@ def _only_number(snap: Snap) -> str | None: __all__ = ["CHECKS", "CHECK_NAMES", "Check", "Outcome", "STATUS_NAMES", "Span", "comparable", "compare_periods", - "compare_queries", "period_span", "reasons", "same_query", "span_coverage", "sql_span", "table_of", - "values_of"] + "compare_queries", "period_complete", "period_span", "reasons", "same_query", "span_coverage", "sql_span", + "table_of", "values_of"] diff --git a/backend/app/analysis/contract.py b/backend/app/analysis/contract.py index 5f1c4b8a..eb038afd 100644 --- a/backend/app/analysis/contract.py +++ b/backend/app/analysis/contract.py @@ -92,7 +92,7 @@ REASONS: tuple[str, ...] = ( CAVEATS: tuple[str, ...] = ( "offset", "simpson", "small_denominator", "record_high", "record_low", "weekday_effect", "multiple_comparisons", "outlier_sensitive", "small_sample", "measure_unlabeled", "null_values", - "headline_basis", "comparability", + "headline_basis", "comparability", "partial_periods", ) # -------------------------------------------------------------------------- diff --git a/backend/app/analysis/periods.py b/backend/app/analysis/periods.py index 2f44d710..5b8a4c9c 100644 --- a/backend/app/analysis/periods.py +++ b/backend/app/analysis/periods.py @@ -6,26 +6,32 @@ 且相邻两期隔整月 → month(月份都在 1、4、7、10 月且隔整季 → quarter);相邻两期都隔 7 天的整数倍 → week(这个日期当 一周的第一天);否则 day。 - 月份:'YYYY-MM'、'YYYYMM'(6 位,月份 01–12)→ month。 -- ISO 周:'YYYY-Www'(2026-W42 是 2026-10-12 到 10-18)→ week。SQLite 的 strftime('%W') 不是 ISO 周号(年初那几天 - 算第 0 周),按它写的期次覆盖的日子会差一周:方法里要求期次写成每周第一天的日期。 +- 周号:'YYYY-Www' → week。同样写成 2026-W42,不同的数据库、不同的格式串算出来的是不同的日子(ISO 周是 10-12 到 10-18, + SQLite 的 strftime('%Y-W%W') 是 10-19 到 10-25),所以起止按 SQL 里写这一列的格式串算(week_scheme 认出周号算法,见 + WEEK_SCHEMES);认不出算法的(方言不知道、只取了一列现成的字符串、年份和周号不配套)起止为空,本期齐不齐核对不了,先后按 + 周号排(同一年里周号连着的算连着,上一年的第 52、53 周和下一年的第 0、1 周算连着)。按年截断的周号算法里,跨年那一周 + 被切成两期,年末、年初那两期不到 7 天(short_days):不是整周,不能当本期、比较期,也不算进历史。 - 季度:'YYYY-Qn'、'YYYYQn' → quarter。 - 整数序号(不是上面几种的整数)→ seq:能排序、能判连续,不知道覆盖哪几天(本期完不完整核对不了)。 带时刻(非零点)的、写法混着的、认不出的都算认不出(period_unreadable)。 -每一期记 ordinal:同一粒度里相邻两期的 ordinal 差 1(日:天数,周:从第一期起第几个 7 天,月:年 × 12 + 月,季:年 × 4 + -季,序号:本身),据此排序、判缺期、按 lag 找比较期。start / end 是这一期覆盖的第一天和最后一天(含),seq 没有。 +每一期记 ordinal:同一粒度里相邻两期的 ordinal 差 1(日:天数,周:日期写的是从第一期起第几个 7 天、周号按它的算法排, +月:年 × 12 + 月,季:年 × 4 + 季,序号:本身),据此排序、判缺期、按 lag 找比较期。start / end 是这一期覆盖的第一天和最后 +一天(含),seq 和认不出算法的周号没有。 """ from __future__ import annotations import calendar import re -from collections.abc import Sequence +from collections.abc import Iterable, Sequence from dataclasses import dataclass from datetime import date, datetime, timedelta +from functools import lru_cache from typing import Any from app.data import coverage +from app.data.names import name_key GRAINS = ("day", "week", "month", "quarter") #: 粒度的叫法:数量词(过去 12 周、12 个月)、每…(每周、每月) @@ -38,6 +44,32 @@ _MONTH = re.compile(r"^(\d{4})-?(\d{2})$") _WEEK = re.compile(r"^(\d{4})-?W(\d{1,2})$", re.I) _QUARTER = re.compile(r"^(\d{4})-?Q([1-4])$", re.I) +#: 「YYYY-Www」的周号算法(week_scheme 按 SQL 认,week_span 按它算一周的起止): +#: - iso:ISO 8601。周一开始,第 1 周是有 4 天以上落在这一年的那周,年份是 ISO 年(年初、年末的几天算到相邻的年) +#: - monday:周一开始,第 1 周从这一年第一个周一起,之前的几天是第 0 周,按年截断(SQLite、C 的 %W) +#: - sunday:周日开始,其余同 monday(%U;MySQL WEEK 模式 0) +#: - monday4:周一开始,第 1 周是有 4 天以上落在这一年的那周,之前的几天是第 0 周,按年截断(MySQL 的 %u,WEEK 模式 1) +#: - sunday_year:周日开始,第 1 周从这一年第一个周日起,年初之前的几天算上一年的最后一周(MySQL 的 %X 配 %V,WEEK 模式 2) +#: - jan1:从 1 月 1 日起每 7 天一周,最后一周只有一两天(PostgreSQL、Oracle to_char 的 WW) +WEEK_SCHEMES = ("iso", "monday", "sunday", "monday4", "sunday_year", "jan1") +#: 各方言格式串里配套的(年份, 周号)写法 → 周号算法。不配套的(%Y 配 ISO 周号、IYYY 配 WW……)同一个标签会落在两段 +#: 日子上,认不出 +_WEEK_FORMATS: dict[str, dict[tuple[str, str], str]] = { + "sqlite": {("%Y", "%W"): "monday", ("%Y", "%U"): "sunday", ("%G", "%V"): "iso"}, + "mysql": {("%Y", "%U"): "sunday", ("%Y", "%u"): "monday4", ("%X", "%V"): "sunday_year", ("%x", "%v"): "iso"}, + "postgres": {("YYYY", "WW"): "jan1", ("IYYY", "IW"): "iso"}, + "oracle": {("YYYY", "WW"): "jan1", ("IYYY", "IW"): "iso"}, +} +#: 各方言格式串里年份、周号的写法(数格式串里有几个年份、几个周号用;方言不同,同一个 %u 意思不同) +_YEAR_MARKS = {"sqlite": ("%Y", "%G"), "mysql": ("%Y", "%X", "%x"), "postgres": ("YYYY", "IYYY"), + "oracle": ("YYYY", "IYYY")} +_WEEK_MARKS = {"sqlite": ("%W", "%U", "%V"), "mysql": ("%U", "%u", "%V", "%v"), "postgres": ("WW", "IW"), + "oracle": ("WW", "IW")} +#: to_char 的格式串里先认长的:IYYY 里有 YYYY,IW、WW 都有 W(一个月里的第几周) +_CHAR_MARKS = ("IYYY", "YYYY", "IW", "WW") +#: 期次列从 CTE、子查询里原样取出时,最多往里找几层 +_NEST_MAX = 4 + @dataclass(frozen=True) class Period: @@ -111,6 +143,219 @@ def _month_of(value: Any) -> date: return date(int(m.group(1)), int(m.group(2)), 1) +# -------------------------------------------------------------------------- +# 周号:按算法算起止、排先后 +# -------------------------------------------------------------------------- + + +def _week_label(value: Any) -> tuple[int, int]: + m = _WEEK.match(str(value).strip()) + assert m is not None + return int(m.group(1)), int(m.group(2)) + + +def _first_weekday(start: date, weekday: int) -> date: + """start 当天或之后的第一个星期几(周一 0 … 周日 6)。""" + return start + timedelta(days=(weekday - start.weekday()) % 7) + + +def week_span(scheme: str, year: int, week: int) -> tuple[date, date] | None: + """按周号算法,year 年第 week 周覆盖的第一天和最后一天(含)。这种算法里没有这一周(ISO 只有 52 周的年份写了第 53 周、 + 1 月 1 日就是周一的年份写了第 0 周……)为 None。""" + if scheme == "iso": + try: + start = date.fromisocalendar(year, week, 1) + except ValueError: + return None + return start, start + timedelta(days=6) + try: + jan1, dec31 = date(year, 1, 1), date(year, 12, 31) + except ValueError: + return None + if scheme == "jan1": + if not 1 <= week <= 53: + return None + start = jan1 + timedelta(days=7 * (week - 1)) + return start, min(start + timedelta(days=6), dec31) + if scheme == "sunday_year": + start = _first_weekday(jan1, 6) + timedelta(days=7 * (week - 1)) + return (start, start + timedelta(days=6)) if week >= 1 and start.year == year else None + if scheme == "monday": + week1 = _first_weekday(jan1, 0) + elif scheme == "sunday": + week1 = _first_weekday(jan1, 6) + elif scheme == "monday4": + week1 = date.fromisocalendar(year, 1, 1) # ISO 第 1 周的周一,可能在上一年的 12 月 + else: + return None + if week == 0: # 第 1 周之前的那几天;第 1 周从 1 月 1 日(或更早)起就没有 + return (jan1, week1 - timedelta(days=1)) if week1 > jan1 else None + start = week1 + timedelta(days=7 * (week - 1)) + if week < 0 or start > dec31: + return None + return max(start, jan1), min(start + timedelta(days=6), dec31) + + +def _week_start(day: date, weekday: int) -> date: + """day 所在那一周(从星期几开始)的第一天。""" + return day - timedelta(days=(day.weekday() - weekday) % 7) + + +def _week_ordinal(scheme: str, span: tuple[date, date], year: int, base: int) -> int: + """一周在这种算法的先后里排第几(相邻两期差 1)。整 7 天一周的按那一周第一天的序数除以 7(同一个星期几开头的日子序数 + 模 7 同余);按年截断的,跨年那一周切成了两期,每过一个不是从这周第一天开始的元旦多一期(base 是这一列里最早的年份)。""" + if scheme == "jan1": + return year * 53 + (span[0] - date(year, 1, 1)).days // 7 + if scheme in ("iso", "sunday_year"): + return span[0].toordinal() // 7 + weekday = 6 if scheme == "sunday" else 0 + splits = sum(1 for y in range(base + 1, year + 1) if date(y, 1, 1).weekday() != weekday) + return _week_start(span[0], weekday).toordinal() // 7 + splits + + +def _loose_ordinals(labels: Iterable[tuple[int, int]]) -> dict[tuple[int, int], int]: + """认不出周号算法时的先后:按(年,周号)排,同一年里周号连着的算连着;上一年的第 52、53 周和下一年的第 0、1 周算连着 + (核对不了跨年那里是不是缺了一周);别的跨年当缺期(隔几期只是估计,至少隔一期)。""" + out: dict[tuple[int, int], int] = {} + prev: tuple[int, int] | None = None + n = 0 + for year, week in sorted(set(labels)): + if prev is not None: + if year == prev[0]: + n += week - prev[1] + elif year == prev[0] + 1 and prev[1] >= 52 and week <= 1: + n += 1 + else: + n += max(2, (year - prev[0]) * 53 + week - prev[1]) + out[(year, week)] = n + prev = (year, week) + return out + + +def short_days(item: Period, grain: str) -> int | None: + """这一期不到整期(按年截断的周号算法里跨年切开的那两期、WW 的最后一周)时有几天;整期、不知道起止的为 None。""" + if grain != "week" or item.start is None or item.end is None: + return None + days = (item.end - item.start).days + 1 + return days if days < 7 else None + + +@lru_cache(maxsize=256) +def week_scheme(sql: str, column: str, dialect: str | None) -> str | None: + """期次列 column 是按哪种周号算法写成「YYYY-Www」的(WEEK_SCHEMES 之一):看最外层 SELECT 里这一列怎么写(从 CTE、子查询 + 里原样取出的往里找),格式化函数(SQLite 的 strftime、MySQL 的 DATE_FORMAT、PostgreSQL / Oracle 的 to_char)的格式串里 + 年份和周号是不是配套的一对(_WEEK_FORMATS),而且只引用了一列。认不出(方言不知道、不是这几个函数、年份和周号不配套或 + 不止一个、只是取了一列现成的字符串)为 None。""" + from sqlglot import exp + + pairs = _WEEK_FORMATS.get(dialect or "") + if pairs is None or not sql: + return None + try: + value = _period_expr(sql, column, dialect) + if value is None: + return None + refs = {(name_key(c.table), name_key(c.name)) for c in value.find_all(exp.Column) + if not isinstance(c.this, exp.Star)} + texts = [_native_format(node, dialect) for node in value.find_all(exp.TimeToStr, exp.ToChar)] # type: ignore[arg-type] + if len(refs) != 1 or not texts or any(t is None for t in texts): + return None + marks = [m for t in texts for m in _format_marks(t, dialect)] # type: ignore[arg-type] + years = [m for m in marks if m in _YEAR_MARKS[dialect]] # type: ignore[index] + weeks = [m for m in marks if m in _WEEK_MARKS[dialect]] # type: ignore[index] + if len(years) != 1 or len(weeks) != 1: + return None + return pairs.get((years[0], weeks[0])) + except Exception: # noqa: BLE001 - 认不出就当不知道算法 + return None + + +def _period_expr(sql: str, column: str, dialect: str | None) -> Any: + """结果列 column 的写法(sqlglot 节点):最外层 SELECT 里那一项;是从 CTE、子查询里原样取出的一列时往里找。""" + from sqlglot import exp + + from app.engine import cell_format as cf + + tree = coverage.parse_sql(sql, dialect) + if not isinstance(tree, exp.Select): + return None + ctes = {name_key(c.alias_or_name): c.this for c in tree.find_all(exp.CTE)} + select, name = tree, column + for _ in range(_NEST_MAX + 1): + item = cf._projection(select, name) + if item is None: + return None + value = cf._unwrap(item.unalias()) + if not isinstance(value, exp.Column) or isinstance(value.this, exp.Star): + return value + inner = _inner_select(select, value, ctes) + if inner is None: + return value + select, name = inner, value.name + return None + + +def _inner_select(select: Any, column: Any, ctes: dict[str, Any]) -> Any: + """外层取的这一列来自哪个 CTE、子查询(那一层的 SELECT);来自实表、认不准是哪个来源的为 None。""" + from sqlglot import exp + + sources = [select.args.get("from_") or select.args.get("from"), *(select.args.get("joins") or [])] + found = [s.this for s in sources if s is not None and getattr(s, "this", None) is not None] + if column.table: + found = [n for n in found if name_key(n.alias_or_name) == name_key(column.table)] + if len(found) != 1: + return None + node = found[0] + if isinstance(node, exp.Subquery): + inner = node.this + elif isinstance(node, exp.Table) and not node.db and name_key(node.name) in ctes: + inner = ctes[name_key(node.name)] + else: + return None + return inner if isinstance(inner, exp.Select) else None + + +def _native_format(node: Any, dialect: str) -> str | None: + """格式化函数的格式串,按这个方言原本的写法(sqlglot 解析时换成了自己的写法,再换回去;Oracle 的 to_char 原样留着)。""" + from sqlglot import exp + from sqlglot.dialects.dialect import Dialect + + fmt = node.args.get("format") + if not isinstance(fmt, exp.Literal) or not fmt.is_string: + return None + if isinstance(node, exp.ToChar): + return str(fmt.this) + text = Dialect.get_or_raise(dialect).generator().format_time(node) + if not text or len(text) < 2 or text[0] != "'" or text[-1] != "'": + return None + return text[1:-1].replace("''", "'") + + +def _format_marks(text: str, dialect: str) -> list[str]: + """格式串里的格式符(按先后):strftime、DATE_FORMAT 是「%」加一个字(%% 是百分号本身);to_char 去掉双引号里的原样文字后 + 按 _CHAR_MARKS 从左往右认(不分大小写)。""" + out: list[str] = [] + i = 0 + if dialect in ("sqlite", "mysql"): + while i < len(text): + if text[i] == "%" and i + 1 < len(text): + if text[i + 1] != "%": + out.append(text[i:i + 2]) + i += 2 + else: + i += 1 + return out + plain = re.sub(r'"[^"]*"', "", text).upper() + while i < len(plain): + mark = next((m for m in _CHAR_MARKS if plain.startswith(m, i)), None) + if mark is not None: + out.append(mark) + i += len(mark) + else: + i += 1 + return out + + def _infer_date_grain(days: Sequence[date]) -> str: """一列日期的粒度(见文件头)。""" ordered = sorted(set(days)) @@ -131,8 +376,10 @@ def _infer_date_grain(days: Sequence[date]) -> str: return "day" -def parse(values: Sequence[Any], grain: str | None = None) -> tuple[list[Period], str] | None: - """一列期次值 → (逐个的 Period,粒度)。认不出(写法不一、带时刻、粒度和写法对不上)返回 None。""" +def parse(values: Sequence[Any], grain: str | None = None, *, + weeks: str | None = None) -> tuple[list[Period], str] | None: + """一列期次值 → (逐个的 Period,粒度)。认不出(写法不一、带时刻、粒度和写法对不上、周号在 weeks 这种算法里没有)返回 None, + 是哪几个值见 unreadable。weeks 是 week_scheme 认出的周号算法:只用于「YYYY-Www」,None 是认不出算法(起止为空)。""" shapes = {_shape(v) for v in values} if not values or None in shapes or len(shapes) != 1: return None @@ -152,16 +399,16 @@ def parse(values: Sequence[Any], grain: str | None = None) -> tuple[list[Period] if shape == "week": if grain not in (None, "week"): return None - out = [] - for v in values: - m = _WEEK.match(str(v).strip()) - assert m is not None - try: - start = date.fromisocalendar(int(m.group(1)), int(m.group(2)), 1) - except ValueError: - return None - out.append(Period(v, start.toordinal() // 7, start, start + timedelta(days=6))) - return out, "week" + labels = [_week_label(v) for v in values] + if weeks is None: + order = _loose_ordinals(labels) + return [Period(v, order[label], None, None) for v, label in zip(values, labels)], "week" + spans = [week_span(weeks, *label) for label in labels] + if any(s is None for s in spans): + return None + base = min(year for year, _ in labels) + return [Period(v, _week_ordinal(weeks, s, label[0], base), s[0], s[1]) # type: ignore[index, arg-type] + for v, label, s in zip(values, labels, spans)], "week" if shape == "quarter": if grain not in (None, "quarter"): return None @@ -196,6 +443,33 @@ def parse(values: Sequence[Any], grain: str | None = None) -> tuple[list[Period] return None +def unreadable(values: Sequence[Any], grain: str | None = None, *, + weeks: str | None = None) -> tuple[str, list[Any]]: + """parse 认不出时,为什么、是哪几个值(按先后、去重):shape 写法认不出(空值、带时刻、不是这几种写法的);mixed 写法混着 + (不是最多那种写法的几个,一样多时以先出现的为准);week 周号在 SQL 的算法里没有这一周;grain 和给的粒度对不上(全部)。""" + shapes = [_shape(v) for v in values] + odd = [v for v, s in zip(values, shapes) if s is None] + kinds = list(dict.fromkeys(shapes)) + if odd or not values: + return "shape", _distinct(odd) + if len(kinds) > 1: + main = max(kinds, key=shapes.count) + return "mixed", _distinct(v for v, s in zip(values, shapes) if s != main) + if kinds[0] == "week" and weeks is not None and grain in (None, "week"): + odd = [v for v in values if week_span(weeks, *_week_label(v)) is None] + if odd: + return "week", _distinct(odd) + return "grain", _distinct(values) + + +def _distinct(values: Iterable[Any]) -> list[Any]: + out: list[Any] = [] + for v in values: + if not any(v == o and type(v) is type(o) for o in out): + out.append(v) + return out + + def calendar_unit(start: date, end: date) -> str | None: """这段日子是不是一个整的日历期:整月 month、整季 quarter、整年 year;都不是为 None。整月对整月算一样长(31 天的 10 月 和 30 天的 9 月环比是通行的比法),不按天数判窗口不等。""" @@ -221,4 +495,5 @@ def weekend_days(start: date, end: date) -> int: return count -__all__ = ["COUNT_WORD", "DEFAULT_LAG", "GRAINS", "PER_WORD", "Period", "calendar_unit", "parse", "weekend_days"] +__all__ = ["COUNT_WORD", "DEFAULT_LAG", "GRAINS", "PER_WORD", "WEEK_SCHEMES", "Period", "calendar_unit", "parse", + "short_days", "unreadable", "week_scheme", "week_span", "weekend_days"] diff --git a/backend/app/analysis/phrases.py b/backend/app/analysis/phrases.py index 0d2c8e11..334d17a2 100644 --- a/backend/app/analysis/phrases.py +++ b/backend/app/analysis/phrases.py @@ -29,7 +29,7 @@ from typing import Any from app.analysis.contract import OP_LABELS from app.analysis.numbers import fixed, to_fraction -#: 附带说明进判定语的先后;不在这里的(measure_unlabeled、headline_basis)只在证据面板 +#: 附带说明进判定语的先后;不在这里的(measure_unlabeled、headline_basis、partial_periods)只在证据面板 PHRASE_ORDER = ("comparability", "simpson", "offset", "outlier_sensitive", "small_sample", "record_high", "record_low", "multiple_comparisons", "weekday_effect", "small_denominator", "null_values") #: 判定语里最多接几条附带说明 @@ -306,6 +306,13 @@ def small_sample_mean_text(group: str, n: int, minimum: int) -> str: UNVERIFIED_COMPLETE = "没能核对本期的数据是否到齐(查不到表里这一列的日期范围)" +UNKNOWN_WEEKS = "没能核对本期的数据是否到齐(期次写成「年-W周号」,认不出 SQL 是按哪种周号算的,不知道是哪几天)" +NO_DATES = "没能核对本期的数据是否到齐(期次不是日期,不知道是哪几天)" + + +def partial_periods_text(count: int) -> str: + """按年截断的周号算法里跨年切开的那几期(不到整周)没有算进历史。""" + return f"有 {count} 期不到整周(跨年被切开),没有算进历史" def with_caveats(main: str, caveats: Sequence[Mapping[str, str]], *, skip: Iterable[str] = ()) -> str: @@ -501,14 +508,15 @@ CRITERION_COMPARABLE = ("只差日期、窗口一样长、是整期、星期构 __all__ = [ - "CRITERION_COMPARABLE", "EMPTY_KEY", "HEADLINE_BASIS", "KEY_JOIN", "LABEL_MAX", "MEASURE_WORDS", - "PHRASE_CAVEATS", "PHRASE_ORDER", "UNVERIFIED_COMPLETE", "WEEKDAY_EFFECT", "amount", "band_text", + "CRITERION_COMPARABLE", "EMPTY_KEY", "HEADLINE_BASIS", "KEY_JOIN", "LABEL_MAX", "MEASURE_WORDS", "NO_DATES", + "PHRASE_CAVEATS", "PHRASE_ORDER", "UNKNOWN_WEEKS", "UNVERIFIED_COMPLETE", "WEEKDAY_EFFECT", "amount", "band_text", "baseline_flat_phrase", "baseline_phrase", "caveat", "change", "clean_label", "comparable_phrase", "criterion_additive", "criterion_baseline", "criterion_differ", "criterion_factor", "criterion_ratio", "criterion_trend", "current_text", "decompose_concentrated", "decompose_dispersed", "decompose_driver", "decompose_flat", "decompose_unclosed", "differ_pair", "differ_phrase", "dims_text", "factor_both", "factor_driver", "key_text", "multiple_text", "names", "no_evidence_phrase", "null_periods_text", - "null_segments_text", "number", "offset_text", "outlier_text", "pct", "pick_label", "points", "ratio_both", + "null_segments_text", "number", "offset_text", "outlier_text", "partial_periods_text", "pct", "pick_label", + "points", "ratio_both", "ratio_change", "ratio_mix", "ratio_rate", "ratio_value", "record_text", "refused", "say", "share_words", "simpson_text", "small_denominator_text", "small_sample_mean_text", "small_sample_rate_text", "threshold_pct", "trend_phrase", "unconfirmed_text", "unlabeled_text", "with_caveats", diff --git a/backend/app/analysis/series.py b/backend/app/analysis/series.py index e3c6c2ac..0a05ac7d 100644 --- a/backend/app/analysis/series.py +++ b/backend/app/analysis/series.py @@ -5,9 +5,11 @@ week / month / quarter,缺省按期次值推断,见 analysis/periods)、la (change 缺省:和比较期比的变化率;level:值本身,给存量、比率用)、lag(和往前第几期比,缺省日数据 7、其余 1;同比写 52 / 12 / 4,历史要相应够长)。 -**前提**(不满足就算不了,原因照实写):期次认得出(period_unreadable)、本期完整(同可比性检查的整期:没过完、数据没到、 -SQL 截掉一部分 → partial_target;表的日期范围查不到时只按运行当天判,附一句核对不了)、本期和比较期之间不缺期(target_gap)、 -比较期不为 0(zero_base)、历史够长(history_short);趋势要最后几期连着(中间缺期 series_gap)。度量格为空的那一期跳过。 +**前提**(不满足就算不了,原因照实写):期次认得出(period_unreadable,原因里举认不出的那几个值)、本期完整(同可比性检查的 +整期:不到整周、没过完、数据没到、SQL 截掉一部分 → partial_target;表的日期范围查不到时只按运行当天判,附一句核对不了;期次 +没有日期、周号认不出算法的不判齐,附一句核对不了)、比较期有值(缺期、那一期的值为空、不到整周 → target_gap,原因照实分开 +写)、比较期不为 0(zero_base)、历史够长(history_short);趋势要最后几期连着(中间缺期、值为空、不到整周 series_gap)。度量 +格为空的那一期跳过;不到整周的那几期(按年截断的周号算法里跨年切开的)不算进历史。 **波动基线**:历史的中位数 m、中位绝对偏差 MAD,σ̂ = 1.4826·MAD,稳健 z = (x − m)/σ̂;常规波动范围 m ± k_warn·σ̂;全程 精确的有理数。σ̂ = 0(历史每期都一样)时本期一样判常规、不一样判明显超出。本期的值(不论 basis)是历史各期里最高 / 最低的 @@ -50,6 +52,20 @@ class Series: points: list[tuple[pd.Period, Fraction]] # 按期次排好、度量不为空的 nulls: int params: dict[str, Any] + periods: dict[int, pd.Period] # 每一期(含度量为空的),按 ordinal + empty: set[int] # 度量为空的那几期的 ordinal + + def whole(self, item: pd.Period) -> bool: + return pd.short_days(item, self.grain) is None + + def missing(self, ordinal: int) -> tuple[str, str] | None: + """某一期拿不到能用的值的原因:(empty 值为空 / short 不到整周, 那一期的写法);真缺了这一期为 None。""" + item = self.periods.get(ordinal) + if item is None: + return None + if ordinal in self.empty: + return "empty", ph.key_text([item.raw]) + return ("short", ph.key_text([item.raw])) if not self.whole(item) else None def _load(raw: Mapping[str, Any], env: Env, used: list[str]) -> Series | Any: @@ -71,29 +87,45 @@ def _load(raw: Mapping[str, Any], env: Env, used: list[str]) -> Series | Any: criterion=criterion) i_period = snap.index(period) values = [row[i_period] if i_period < len(row) else None for row in snap.rows] - parsed = pd.parse(values, grain) + weeks = pd.week_scheme(snap.sql, period, snap.dialect) + parsed = pd.parse(values, grain, weeks=weeks) if parsed is None: - sample = "、".join(dict.fromkeys(ph.key_text([v]) for v in values[:3])) - return refuse(env, params, used, "period_unreadable", - f"期次列「{period}」的值认不出是哪一期({sample}):要日期、YYYY-MM、YYYY-Www、YYYY-Qn 或整数序号", + return refuse(env, params, used, "period_unreadable", unreadable_text(period, values, grain, weeks), criterion=criterion) items, grain = parsed params["grain"] = grain - seen: set[int] = set() + periods: dict[int, pd.Period] = {} + empty: set[int] = set() points: list[tuple[pd.Period, Fraction]] = [] - nulls = 0 for r, item in enumerate(items): - if item.ordinal in seen: + if item.ordinal in periods: raise CallError("duplicate_rows", ph.say(f"{snap.ref} 里同一期有不止一行:在 SQL 里按 ", column_label(env, snap, period), " 汇总,每期一行")) - seen.add(item.ordinal) + periods[item.ordinal] = item value = snap.number(r, measure, "度量列") if value is None: - nulls += 1 + empty.add(item.ordinal) continue points.append((item, value)) points.sort(key=lambda p: p[0].ordinal) - return Series(snap, period, measure, grain, label, points, nulls, params) + return Series(snap, period, measure, grain, label, points, len(empty), params, periods, empty) + + +#: 给了粒度、期次对不上时的说法 +_GRAIN_WORD = {"day": "按天", "week": "按周", "month": "按月", "quarter": "按季度", "seq": "按序号"} + + +def unreadable_text(period: str, values: list[Any], grain: str | None, weeks: str | None) -> str: + """期次认不出的原因:举出认不出的那几个值(最多三个),按为什么认不出分开说(periods.unreadable)。""" + kind, odd = pd.unreadable(values, grain, weeks=weeks) + shown = "、".join("空值" if v is None else ph.key_text([v]) for v in odd[:3]) + ("等" if len(odd) > 3 else "") + if kind == "mixed": + return ph.say(f"期次列「{period}」的写法不一:", shown, "和其余的不是同一种写法") + if kind == "week": + return ph.say(f"期次列「{period}」里的", shown, "按 SQL 里的周号算法没有这一周") + if kind == "grain": + return ph.say(f"期次列「{period}」的值(", shown, f")对不上{_GRAIN_WORD.get(str(grain), '给的')}的粒度") + return f"期次列「{period}」的值认不出是哪一期({shown}):要日期、YYYY-MM、YYYY-Www、YYYY-Qn 或整数序号" def _criterion(env: Env, n: int | None, unit: str) -> str: @@ -102,13 +134,11 @@ def _criterion(env: Env, n: int | None, unit: str) -> str: return ph.criterion_trend(env.settings, n, unit) -def _completeness(env: Env, series: Series, item: pd.Period) -> tuple[str, str | None]: - """本期齐不齐(同可比性检查的整期):(pass / short / unknown, 不齐的那一句)。""" +def _completeness(env: Env, series: Series, item: pd.Period) -> tuple[str, str | None, str | None]: + """本期齐不齐(同可比性检查的整期,comparable.period_complete):(pass / short / unknown, 不齐的那一句, 核对不了时的附带说明)。""" series.params["today"] = today_param(env) - if item.start is None: - return "unknown", None - span = cmp.period_span("本期", series.snap, series.period, item, series.grain, series.measure, env.facts, env.book) - return cmp.span_coverage(span, env.today, env.book) # type: ignore[arg-type] + return cmp.period_complete(series.snap, series.period, item, series.grain, series.measure, env.facts, env.book, + env.today) # type: ignore[arg-type] # -------------------------------------------------------------------------- @@ -144,18 +174,19 @@ def baseline(raw: Mapping[str, Any], env: Env) -> Any: target, x_level = found params["target"] = target.raw caveats: list[dict[str, str]] = [] - status, why = _completeness(env, series, target) + status, why, note = _completeness(env, series, target) if status == "short": return refuse(env, params, used, "partial_target", f"{why},和完整的{per}比没有意义", criterion=criterion) - if status == "unknown" and target.start is not None: - caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE)) - by_ordinal = {p.ordinal: v for p, v in series.points} - before = [(p, v) for p, v in series.points if p.ordinal < target.ordinal] + if status == "unknown" and note: + caveats.append(ph.caveat("comparability", note)) + # 不到整周的那几期(跨年切开的)既不当比较期、也不算进历史 + by_ordinal = {p.ordinal: v for p, v in series.points if series.whole(p)} + before = [(p, v) for p, v in series.points if p.ordinal < target.ordinal and series.whole(p)] + cut = sum(1 for p, _ in series.points if p.ordinal < target.ordinal and not series.whole(p)) if basis == "change": prev = by_ordinal.get(target.ordinal - lag) if prev is None: - return refuse(env, params, used, "target_gap", - f"本期和它的比较期(往前第 {lag} {'期' if series.grain == 'seq' else unit})之间缺期,没有比较期的值", + return refuse(env, params, used, "target_gap", _compare_gap(series, target.ordinal - lag, lag, unit), criterion=criterion) if prev == 0: return refuse(env, params, used, "zero_base", ph.say("比较期的", series.label, "为 0,算不了变化率"), @@ -190,6 +221,8 @@ def baseline(raw: Mapping[str, Any], env: Env) -> Any: caveats.append(ph.caveat("record_low", ph.record_text(False, series.label, n, unit))) if series.nulls: caveats.append(ph.caveat("null_values", ph.null_periods_text(series.nulls, series.label))) + if cut: + caveats.append(ph.caveat("partial_periods", ph.partial_periods_text(cut))) current = ph.current_text(x, basis=basis, lag=lag, grain=series.grain) if sigma == 0: shown = ph.pct(m) if basis == "change" else ph.amount(m) @@ -213,6 +246,17 @@ def baseline(raw: Mapping[str, Any], env: Env) -> Any: phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats) +def _compare_gap(series: Series, ordinal: int, lag: int, unit: str) -> str: + """比较期拿不到值的原因,照实分开说:那一期的值为空、那一期不到整周、真缺了那一期。""" + found = series.missing(ordinal) + if found is not None and found[0] == "empty": + return ph.say("比较期(", found[1], ")的", series.label, "为空,没有比较期的值") + if found is not None: + days = pd.short_days(series.periods[ordinal], series.grain) + return ph.say("比较期(", found[1], f")只有 {days} 天,不是整周,和它比没有意义") + return f"本期和它的比较期(往前第 {lag} {'期' if series.grain == 'seq' else unit})之间缺期,没有比较期的值" + + # -------------------------------------------------------------------------- # 趋势 # -------------------------------------------------------------------------- @@ -245,15 +289,15 @@ def trend(raw: Mapping[str, Any], env: Env) -> Any: return refuse(env, params, used, "history_short", f"只有 0 期,至少要 {minimum} 期", criterion=criterion) last = series.points[-1][0] caveats: list[dict[str, str]] = [] - status, why = _completeness(env, series, last) + status, why, note = _completeness(env, series, last) if status == "short": return refuse(env, params, used, "partial_target", f"{why},和完整的{per}比没有意义", criterion=criterion) - if status == "unknown" and last.start is not None: - caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE)) + if status == "unknown" and note: + caveats.append(ph.caveat("comparability", note)) cap = int(env.settings["max_history"]) + 1 tail = [series.points[-1]] for point in reversed(series.points[:-1]): - if len(tail) >= cap or point[0].ordinal != tail[-1][0].ordinal - 1: + if len(tail) >= cap or point[0].ordinal != tail[-1][0].ordinal - 1 or not series.whole(point[0]): break tail.append(point) tail.reverse() @@ -263,7 +307,8 @@ def trend(raw: Mapping[str, Any], env: Env) -> Any: gap = len(series.points) > n and len(tail) < cap if gap: return refuse(env, params, used, "series_gap", - f"最后连着的只有 {n} 期,再往前缺期(至少要连着的 {minimum} 期)", criterion=criterion) + f"最后连着的只有 {n} 期,{_tail_gap(series, tail[0][0].ordinal - 1)}(至少要连着的 {minimum} 期)", + criterion=criterion) return refuse(env, params, used, "history_short", f"只有 {n} 期,至少要 {minimum} 期", criterion=criterion) ys = [v for _, v in tail] xs = [Fraction(i) for i in range(n)] @@ -302,4 +347,14 @@ def trend(raw: Mapping[str, Any], env: Env) -> Any: phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats) -__all__ = ["MAD_SCALE", "baseline", "run_length", "trend"] +def _tail_gap(series: Series, ordinal: int) -> str: + """最后连着的一段往前断开的原因:那一期的值为空、不到整周,或者真缺了那一期。""" + found = series.missing(ordinal) + if found is not None and found[0] == "empty": + return ph.say("再往前的", found[1], "那一期的", series.label, "为空") + if found is not None: + return ph.say("再往前的", found[1], f"只有 {pd.short_days(series.periods[ordinal], series.grain)} 天,不是整周") + return "再往前缺期" + + +__all__ = ["MAD_SCALE", "baseline", "run_length", "trend", "unreadable_text"] diff --git a/backend/tests/test_analysis_comparable.py b/backend/tests/test_analysis_comparable.py index 645c3d9d..caf8d433 100644 --- a/backend/tests/test_analysis_comparable.py +++ b/backend/tests/test_analysis_comparable.py @@ -181,3 +181,26 @@ async def test_a_masked_date_column_is_neither_used_nor_probed(): period("Q2", "2026-10-12", "2026-10-19", masked=["order_date"]), probe=probe, measure="gmv") assert probe.calls == [] and result["probes"] == [] assert result["verdict"]["code"] == "comparable" and "窗口长度" in result["phrase"] + + +async def test_one_snapshot_of_sqlite_weeks_uses_the_real_days_of_each_week(): + # SQLite 的 strftime('%Y-W%W'):2026-W42 是 10-19 到 10-25(运行当天 10-21 还没过完),跨年的 2025-W52、2026-W00 只有 3 天、4 天 + sql = "SELECT strftime('%Y-W%W', order_date) AS week, COUNT(*) AS orders FROM orders GROUP BY week" + rows = [["2025-W52", 300], ["2026-W00", 400], ["2026-W01", 700], ["2026-W41", 700], ["2026-W42", 200]] + s = snap("Q1", sql, ["week", "orders"], rows, types={"week": "text", "orders": "number"}) + probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")}) + now = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="week", base="2026-W41", + target="2026-W42") + assert now["verdict"]["code"] == "incomparable" + assert now["phrase"] == "两期不可比:本期数据只到 2026-10-20,还差 5 天" + assert now["values"]["target_start"] == "2026-10-19" and now["values"]["target_end"] == "2026-10-25" + cut = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="week", base="2025-W52", + target="2026-W00") + assert cut["verdict"]["code"] == "incomparable" + assert "两段的天数不同:3 天对 4 天" in cut["phrase"] + assert "本期只有 2026-01-01 到 2026-01-04 这 4 天,不是整周" in comparable_reasons(cut) + assert checks(cut)["星期构成"] == "提醒" + # 不知道方言(认不出周号算法):窗口、整期、星期构成都核对不了,不判可比以外的结论 + plain = snap("Q1", sql, ["week", "orders"], rows, types={"week": "text", "orders": "number"}) + unknown = await analyze("comparable", snaps=[plain], query="Q1", period="week", base="2026-W41", target="2026-W42") + assert unknown["phrase"] == "两期可比,窗口长度、整期、星期构成核对不了" diff --git a/backend/tests/test_analysis_periods.py b/backend/tests/test_analysis_periods.py index eee79dde..241f3f1a 100644 --- a/backend/tests/test_analysis_periods.py +++ b/backend/tests/test_analysis_periods.py @@ -1,5 +1,6 @@ """期次(app/analysis/periods.py):日期、YYYY-MM、YYYY-Www、YYYY-Qn、整数序号认得出、能排序、知道覆盖哪几天;粒度按值推断; -写法混着、带时刻的认不出。""" +写法混着、带时刻的认不出。周号按 SQL 里的格式串认算法(SQLite 的 %W / %U / %G-%V、MySQL 的 %U / %u / %X-%V / %x-%v、to_char +的 WW / IW),按算法算真实起止;认不出算法的起止为空。""" from __future__ import annotations from datetime import date @@ -26,10 +27,10 @@ def test_labels_for_months_weeks_quarters_and_sequences(): assert grain == "month" and items[1].ordinal - items[0].ordinal == 1 and items[1].end == date(2026, 10, 31) items, grain = pd.parse([202609, 202610]) assert grain == "month" - items, grain = pd.parse(["2026-W41", "2026-W42", "2026-W01"]) + items, grain = pd.parse(["2026-W41", "2026-W42", "2026-W01"], weeks="iso") assert grain == "week" and items[1].start == date(2026, 10, 12) and items[1].end == date(2026, 10, 18) assert items[1].ordinal - items[0].ordinal == 1 - items, grain = pd.parse(["2025-W52", "2026-W01"]) # 跨年也连着 + items, grain = pd.parse(["2025-W52", "2026-W01"], weeks="iso") # 跨年也连着 assert items[1].ordinal - items[0].ordinal == 1 items, grain = pd.parse(["2026Q3", "2026-Q4"]) assert grain == "quarter" and items[1].start == date(2026, 10, 1) @@ -55,3 +56,75 @@ def test_calendar_units_and_weekend_days(): assert pd.weekend_days(date(2026, 10, 16), date(2026, 10, 18)) == 2 # 周五到周日 assert pd.weekend_days(date(2026, 10, 12), date(2026, 10, 14)) == 0 assert pd.weekend_days(date(2026, 10, 1), date(2026, 10, 31)) == 9 + + +# -------------------------------------------------------------------------- +# 周号:按 SQL 认算法 +# -------------------------------------------------------------------------- + +WEEK_SQL = { + ("sqlite", "SELECT strftime('%Y-W%W', day) AS week, SUM(amount) AS gmv FROM orders GROUP BY week"): "monday", + ("sqlite", "SELECT strftime('%Y-W%U', day) AS week FROM orders GROUP BY 1"): "sunday", + ("sqlite", "SELECT strftime('%G-W%V', day) AS week FROM orders GROUP BY 1"): "iso", + ("sqlite", "SELECT strftime('%Y', day) || '-W' || strftime('%W', day) AS week FROM orders GROUP BY 1"): "monday", + ("sqlite", "WITH w AS (SELECT strftime('%Y-W%W', day) AS wk, amount FROM orders) " + "SELECT wk AS week, SUM(amount) AS gmv FROM w GROUP BY 1"): "monday", + ("sqlite", "SELECT t.wk AS week FROM (SELECT strftime('%G-W%V', day) AS wk FROM orders) AS t GROUP BY 1"): "iso", + ("mysql", "SELECT DATE_FORMAT(day, '%x-W%v') AS week FROM orders GROUP BY 1"): "iso", + ("mysql", "SELECT DATE_FORMAT(day, '%X-W%V') AS week FROM orders GROUP BY 1"): "sunday_year", + ("mysql", "SELECT DATE_FORMAT(day, '%Y-W%u') AS week FROM orders GROUP BY 1"): "monday4", + ("mysql", "SELECT DATE_FORMAT(day, '%Y-W%U') AS week FROM orders GROUP BY 1"): "sunday", + ("postgres", "SELECT to_char(day, 'IYYY-\"W\"IW') AS week FROM orders GROUP BY 1"): "iso", + ("postgres", "SELECT to_char(day, 'YYYY-\"W\"WW') AS week FROM orders GROUP BY 1"): "jan1", + ("oracle", "SELECT TO_CHAR(day, 'IYYY-\"W\"IW') AS week FROM orders GROUP BY TO_CHAR(day, 'IYYY-\"W\"IW')"): "iso", + ("oracle", "SELECT to_char(day, 'YYYY\"W\"WW') AS week FROM orders"): "jan1", +} + + +def test_the_week_algorithm_is_read_from_the_sql(): + for (dialect, sql), scheme in WEEK_SQL.items(): + assert pd.week_scheme(sql, "week", dialect) == scheme, (dialect, sql) + + +def test_week_algorithms_that_cannot_be_told_are_unknown(): + # 年份和周号不配套:同一个标签会落在两段日子上 + assert pd.week_scheme("SELECT strftime('%Y-W%V', day) AS week FROM orders", "week", "sqlite") is None + assert pd.week_scheme("SELECT DATE_FORMAT(day, '%Y-W%v') AS week FROM orders", "week", "mysql") is None + assert pd.week_scheme("SELECT to_char(day, 'YYYY-\"W\"IW') AS week FROM orders", "week", "postgres") is None + # MySQL 的 %W 是星期几的名字,不是周号;方言不知道;取的是现成的一列;两个日期列 + assert pd.week_scheme("SELECT DATE_FORMAT(day, '%Y-W%W') AS week FROM orders", "week", "mysql") is None + assert pd.week_scheme("SELECT strftime('%Y-W%W', day) AS week FROM orders", "week", None) is None + assert pd.week_scheme("SELECT week, SUM(amount) AS gmv FROM weekly GROUP BY week", "week", "sqlite") is None + assert pd.week_scheme("SELECT strftime('%Y', a) || '-W' || strftime('%W', b) AS week FROM t", "week", "sqlite") \ + is None + assert pd.week_scheme("SELECT strftime('%Y-W%W', day) AS wk FROM orders", "week", "sqlite") is None + + +def test_sqlite_monday_weeks_cut_at_new_year(): + items, grain = pd.parse(["2025-W52", "2026-W00", "2026-W01", "2026-W42"], weeks="monday") + spans = [(i.start, i.end) for i in items] + assert spans == [(date(2025, 12, 29), date(2025, 12, 31)), (date(2026, 1, 1), date(2026, 1, 4)), + (date(2026, 1, 5), date(2026, 1, 11)), (date(2026, 10, 19), date(2026, 10, 25))] + assert [i.ordinal - items[0].ordinal for i in items[:3]] == [0, 1, 2] # 跨年切开的两期也连着 + assert [pd.short_days(i, grain) for i in items] == [3, 4, None, None] + # 1 月 1 日就是周一的年份没有第 0 周:认不出,原因里举的是它 + assert pd.parse(["2024-W00", "2024-W01"], weeks="monday") is None + assert pd.unreadable(["2024-W00", "2024-W01"], weeks="monday") == ("week", ["2024-W00"]) + # ISO 只有 52 周的年份没有第 53 周 + assert pd.parse(["2026-W53"], weeks="iso") is not None and pd.parse(["2025-W53"], weeks="iso") is None + + +def test_weeks_of_an_unknown_algorithm_have_no_dates_but_keep_their_order(): + items, grain = pd.parse(["2025-W51", "2025-W52", "2026-W00", "2026-W01", "2026-W03"]) + assert grain == "week" and all(i.start is None and i.end is None for i in items) + assert [i.ordinal - items[0].ordinal for i in items] == [0, 1, 2, 3, 5] # 同一年里隔了一周就是缺期 + items, _ = pd.parse(["2025-W52", "2026-W01"]) # 上一年最后一周接下一年第一周 + assert items[1].ordinal - items[0].ordinal == 1 + items, _ = pd.parse(["2025-W40", "2026-W01"]) + assert items[1].ordinal - items[0].ordinal > 1 + + +def test_unreadable_values_are_named(): + assert pd.unreadable(["2026-W40", "上周", "2026-W41", None]) == ("shape", ["上周", None]) + assert pd.unreadable(["2026-W40", "2026-W41", "2026-10-12"]) == ("mixed", ["2026-10-12"]) + assert pd.unreadable(["2026-10", "2026-11"], "week") == ("grain", ["2026-10", "2026-11"]) diff --git a/backend/tests/test_analysis_series.py b/backend/tests/test_analysis_series.py index 4f1b8dcf..671598a3 100644 --- a/backend/tests/test_analysis_series.py +++ b/backend/tests/test_analysis_series.py @@ -2,6 +2,7 @@ lag 7、新高新低;上升 / 下降 / 无趋势、连续期数、缺期。每个用例都顺带核对复算和运行逐字段相同。""" from __future__ import annotations +import sqlite3 from datetime import date, timedelta from fractions import Fraction @@ -210,3 +211,105 @@ async def test_daily_trends_warn_about_weekday_effects(): assert any(c["code"] == "weekday_effect" and c["text"] == "日数据有星期效应,看趋势建议按周汇总" for c in result["caveats"]) assert "日数据有星期效应" in result["phrase"] + + +# -------------------------------------------------------------------------- +# 周号按 SQL 的算法算起止(SQLite 的 strftime('%W') 不是 ISO 周) +# -------------------------------------------------------------------------- + +#: SQLite 按 %W 写的周:第 1 周从这一年第一个周一起,之前的几天是第 0 周,跨年那一周被切成两期 +WEEKNO = "SELECT strftime('%Y-W%W', order_date) AS week, COUNT(*) AS orders FROM orders GROUP BY week ORDER BY week" + + +def sqlite_weeks(first, last, sql=WEEKNO): + """真用 SQLite 按 sql 汇总:每天 100 笔订单,从 first 到 last(含)。""" + db = sqlite3.connect(":memory:") + db.execute("CREATE TABLE orders (order_date TEXT)") + day = first + while day <= last: + db.executemany("INSERT INTO orders VALUES (?)", [(day.isoformat(),)] * 100) + day += timedelta(days=1) + rows = [list(r) for r in db.execute(sql)] + db.close() + return rows + + +def weekly(rows, sql=WEEKNO, **kw): + return snap("Q1", sql, ["week", "orders"], rows, types={"week": "text", "orders": "number"}, **kw) + + +async def test_a_sqlite_week_that_is_not_over_is_not_a_whole_week(): + # 运行当天 10-21、数据到 10-20:SQLite 的 2026-W42 是 10-19 到 10-25,只有两天的数据(ISO 的 W42 是 10-12 到 10-18) + rows = sqlite_weeks(date(2026, 6, 1), date(2026, 10, 20)) + assert rows[-1] == ["2026-W42", 200] + probe = fixed_prober({("orders", "order_date"): ("2026-06-01", "2026-10-20")}) + result = await analyze("baseline", snaps=[weekly(rows)], probe=probe, query="Q1", period="week", measure="orders", + label="订单数") + assert result["reason_code"] == "partial_target" + assert result["reason"] == "本期数据只到 2026-10-20,还差 5 天,和完整的周比没有意义" + assert result["params"]["target"] == "2026-W42" + whole = await analyze("baseline", snaps=[weekly(rows)], probe=probe, query="Q1", period="week", measure="orders", + label="订单数", target="2026-W41") + assert whole["status"] == "ok" and whole["verdict"]["code"] == "within_band" + # 不知道方言(认不出算法):起止为空,本期齐不齐核对不了,绝不判齐 + unknown = await analyze("baseline", snaps=[weekly(rows)], query="Q1", period="week", measure="orders", + label="订单数", target="2026-W41") + note = next(c["text"] for c in unknown["caveats"] if c["code"] == "comparability") + assert note == "没能核对本期的数据是否到齐(期次写成「年-W周号」,认不出 SQL 是按哪种周号算的,不知道是哪几天)" + + +async def test_sqlite_weeks_across_new_year_are_readable_and_the_cut_weeks_are_left_out(): + rows = sqlite_weeks(date(2025, 9, 29), date(2026, 3, 1)) + assert ["2025-W52", 300] in rows and ["2026-W00", 400] in rows + probe = fixed_prober({("orders", "order_date"): ("2025-09-29", "2026-03-01")}) + ask = {"snaps": [weekly(rows)], "probe": probe, "query": "Q1", "period": "week", "measure": "orders", + "label": "订单数"} + trend = await analyze("trend", **ask) + # 最后连着的整周是 2026-W01 到 W08:再往前的 2026-W00 只有 4 天 + assert trend["status"] == "ok" and trend["values"]["n"] == "8" and trend["verdict"]["code"] == "no_trend" + assert [r[0] for r in trend["table"]["rows"]][0] == "2026-W01" + year_end = await analyze("baseline", **ask, target="2026-W00") + assert year_end["reason_code"] == "partial_target" + assert year_end["reason"] == "本期只有 2026-01-01 到 2026-01-04 这 4 天,不是整周,和完整的周比没有意义" + after = await analyze("baseline", **ask, target="2026-W01") + assert after["reason_code"] == "target_gap" + assert after["reason"] == "比较期(2026-W00)只有 4 天,不是整周,和它比没有意义" + later = await analyze("baseline", **ask, target="2026-W03") + # 历史:2025-W40 到 W51 各和上一周比(12 个),2026-W02 和 W01 比(1 个);W01 和 W00、W00 和 2025-W52 都不算 + assert later["status"] == "ok" and later["values"]["n_history"] == "13" + assert {"code": "partial_periods", "text": "有 2 期不到整周(跨年被切开),没有算进历史"} in later["caveats"] + assert "跨年" not in later["phrase"] + short = await analyze("trend", **{**ask, "snaps": [weekly([r for r in rows if r[0] <= "2026-W03"])]}) + assert short["reason_code"] == "series_gap" + assert short["reason"] == "最后连着的只有 3 期,再往前的 2026-W00 只有 4 天,不是整周(至少要连着的 6 期)" + + +async def test_unreadable_periods_name_the_values_that_cannot_be_read(): + rows = [[f"2026-W{w:02d}", 100] for w in range(30, 42)] + [["上周", 100]] + result = await baseline(rows) + assert result["reason_code"] == "period_unreadable" + assert result["reason"] == "期次列「week」的值认不出是哪一期(上周):要日期、YYYY-MM、YYYY-Www、YYYY-Qn 或整数序号" + mixed = await baseline([[f"2026-W{w:02d}", 100] for w in range(30, 42)] + [["2026-10-12", 100]]) + assert mixed["reason"] == "期次列「week」的写法不一:2026-10-12 和其余的不是同一种写法" + missing = await analyze("baseline", snaps=[weekly([["2024-W00", 1], ["2024-W01", 2]])], probe=fixed_prober(), + query="Q1", period="week", measure="orders") + assert missing["reason"] == "期次列「week」里的 2024-W00 按 SQL 里的周号算法没有这一周" + + +async def test_an_empty_comparison_period_is_said_as_empty_not_as_a_gap(): + rows = weeks(NOISE + [100, 101]) + rows[-2][1] = None # 本期的上一周有这一行,值为空 + result = await baseline(rows) + assert result["reason_code"] == "target_gap" + assert result["reason"] == f"比较期({rows[-2][0]})的订单数为空,没有比较期的值" + tail = weeks([100 + i for i in range(10)]) + tail[-4][1] = None + broken = await trend(tail) + assert broken["reason"] == f"最后连着的只有 3 期,再往前的 {tail[-4][0]} 那一期的线上销售额为空(至少要连着的 6 期)" + + +async def test_sequence_periods_say_completeness_cannot_be_checked(): + rows = [[i + 1, v] for i, v in enumerate([10, 12, 11, 13, 9, 10, 12, 11, 16])] + result = await baseline(rows, basis="level") + assert {"code": "comparability", "text": "没能核对本期的数据是否到齐(期次不是日期,不知道是哪几天)"} \ + in result["caveats"] From 0acf9fb3a82d9c857052f1f3f098bbdecf54705b Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 15:37:48 +0800 Subject: [PATCH 02/13] =?UTF-8?q?=E5=8F=AB=E6=B3=95=E6=B6=88=E6=AF=92?= =?UTF-8?q?=E5=81=9A=E5=88=B0=E4=B8=8D=E5=86=8D=E5=8F=98=E5=8C=96=E3=80=81?= =?UTF-8?q?=E6=88=AA=E5=88=B0=E7=AC=AC=E4=B8=80=E4=B8=AA=E5=88=86=E5=8F=A5?= =?UTF-8?q?=E6=A0=87=E7=82=B9=E3=80=81=E4=B8=8D=E8=AE=B8=E5=A4=B9=E5=B8=A6?= =?UTF-8?q?=E5=88=A4=E6=96=AD=E8=AF=8D=EF=BC=88=E5=A4=8D=E6=A0=B8=20R12?= =?UTF-8?q?=E3=80=81R18=EF=BC=89=EF=BC=9Aclean=5Flabel=20=E5=8F=8D?= =?UTF-8?q?=E5=A4=8D=E5=8E=BB=E5=BC=95=E7=94=A8=E6=A0=87=E8=AE=B0=E3=80=81?= =?UTF-8?q?=E6=95=B0=E5=AD=97=E3=80=81=E7=AC=A6=E5=8F=B7=E3=80=81=E7=A9=BA?= =?UTF-8?q?=E6=8B=AC=E5=8F=B7=E7=9B=B4=E5=88=B0=E4=B8=8D=E5=8F=98=EF=BC=88?= =?UTF-8?q?=E3=80=8C=E9=94=80=E5=94=AE=E9=A2=9D[=20[v:Q1.r0.gmv]=20]?= =?UTF-8?q?=E3=80=8D=E5=8E=BB=E5=AE=8C=E6=95=B0=E5=AD=97=E6=8B=BC=E5=87=BA?= =?UTF-8?q?=E7=9A=84=E3=80=8C[[v:Q.r.gmv]]=E3=80=8D=E6=8E=A5=E7=9D=80?= =?UTF-8?q?=E5=8E=BB=E6=8E=89=EF=BC=89=EF=BC=8C=E5=86=8D=E5=8E=BB=E6=8E=89?= =?UTF-8?q?=E5=89=A9=E4=B8=8B=E7=9A=84=E6=96=B9=E6=8B=AC=E5=8F=B7=E3=80=81?= =?UTF-8?q?=E6=88=AA=E5=88=B0=E7=AC=AC=E4=B8=80=E4=B8=AA=E5=88=86=E5=8F=A5?= =?UTF-8?q?=E6=A0=87=E7=82=B9=EF=BC=88=EF=BC=8C=E3=80=82=EF=BC=9B=EF=BC=9A?= =?UTF-8?q?=EF=BC=81=EF=BC=9F=E3=80=81,.;:!=3F=EF=BC=89=E4=B9=8B=E5=89=8D?= =?UTF-8?q?=E3=80=81=E5=8E=BB=E6=8E=89=E6=88=AA=E5=AE=8C=E6=B2=A1=E9=85=8D?= =?UTF-8?q?=E4=B8=8A=E5=AF=B9=E7=9A=84=E5=B7=A6=E6=8B=AC=E5=8F=B7=EF=BC=8C?= =?UTF-8?q?=E6=95=B4=E4=B8=AA=E8=BF=87=E7=A8=8B=E5=86=8D=E5=81=9A=E5=88=B0?= =?UTF-8?q?=E4=B8=8D=E5=8F=98=EF=BC=8C=E6=B6=88=E8=BF=87=E7=9A=84=E5=86=8D?= =?UTF-8?q?=E6=B6=88=E4=B8=80=E9=81=8D=E4=B8=8D=E5=8F=98=E2=80=94=E2=80=94?= =?UTF-8?q?=E5=A4=8D=E7=AE=97=E6=8B=BF=E8=AE=B0=E4=B8=8B=E7=9A=84=E5=8F=AB?= =?UTF-8?q?=E6=B3=95=E5=86=8D=E6=B6=88=EF=BC=8C=E5=88=A4=E5=AE=9A=E8=AF=AD?= =?UTF-8?q?=E9=80=90=E5=AD=97=E7=9B=B8=E5=90=8C=E3=80=82=E6=B6=88=E5=AE=8C?= =?UTF-8?q?=E7=9A=84=E5=8F=AB=E6=B3=95=E9=87=8C=E6=9C=89=E8=AF=B4=E6=B3=95?= =?UTF-8?q?=E8=A1=A8=E7=9A=84=E5=88=A4=E6=96=AD=E8=AF=8D=EF=BC=88engine/an?= =?UTF-8?q?alysis=5Fclaims.scan=EF=BC=8C=E5=9C=A8=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E9=87=8C=E5=AF=BC=E5=85=A5=EF=BC=89=E5=B0=B1=E6=95=B4=E4=B8=AA?= =?UTF-8?q?=E4=B8=8D=E8=A6=81=EF=BC=8C=E9=80=80=E5=9B=9E=E4=B8=8B=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E6=9D=A5=E6=BA=90=EF=BC=9B=E7=9B=AE=E5=BD=95=E5=8F=AB?= =?UTF-8?q?=E6=B3=95=E3=80=81=E5=BD=93=E5=8F=AB=E6=B3=95=E7=94=A8=E7=9A=84?= =?UTF-8?q?=E5=88=97=E5=90=8D=E5=90=8C=E4=B8=80=E4=B8=AA=E5=87=BD=E6=95=B0?= =?UTF-8?q?=EF=BC=8C=E5=90=8C=E6=A0=B7=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/phrases.py | 65 +++++++++++++++++++++--- backend/tests/test_analysis_decompose.py | 13 +++++ backend/tests/test_analysis_phrases.py | 22 ++++++++ 3 files changed, 93 insertions(+), 7 deletions(-) diff --git a/backend/app/analysis/phrases.py b/backend/app/analysis/phrases.py index 334d17a2..c379364a 100644 --- a/backend/app/analysis/phrases.py +++ b/backend/app/analysis/phrases.py @@ -11,9 +11,11 @@ - 百分点:比率的变化(2.1% 到 2.5% 是上升 0.4 个百分点)。 **label 消毒**(§7.3):判定语里的度量叫法、因子名、维度叫法是 agent 或数据给的。agent 给的 label(以及目录里的叫法、 -当叫法用的列名)去掉数字、百分号、单位符号、引用标记,不超过 20 字,消毒后为空就退回下一个来源:不然模型可以把 -「销售额(同比 +30%)」塞进 label,借系统渲染绕过裸数字检测。去数字按出具校验抽数字的同一个函数(engine/issuance 的 -extract_numbers:阿拉伯数字、中文数字、带万 / 亿 / 成 / 倍的写法),再去掉剩下的数字字符。维度值是数据,照原样。 +当叫法用的列名)去掉数字、百分号、单位符号、引用标记,截到第一个分句标点之前,不超过 20 字;里面有说法表的判断词的整个 +不要;消毒后为空就退回下一个来源:不然模型可以把「销售额(同比 +30%)」「销售额,主要来自华东」塞进 label,借系统渲染绕过 +裸数字检测和说法核对(判定语不受说法核对)。去数字按出具校验抽数字的同一个函数(engine/issuance 的 extract_numbers:阿拉伯 +数字、中文数字、带万 / 亿 / 成 / 倍的写法),再去掉剩下的数字字符。消毒反复做到不再变化:消过的再消一遍不变,复算拿记下的 +叫法再消,判定语逐字相同。维度值是数据,照原样。 **附带说明**:判定语后面用「;」接,最多两条(按 PHRASE_ORDER 的先后),其余只在证据面板。度量类型没标、以各段之和为 总数这类方法上的交代不进判定语。 @@ -168,13 +170,63 @@ _DIGITS = re.compile(r"[0-90-9]") _SYMBOLS = re.compile(r"[%%‰‱¥¥$$€£₩℃°++\-−–±×÷==<><>~~^*]") _EMPTY_BRACKETS = re.compile(r"[((【\[]\s*[))】\]]") _EDGE_PUNCT = ",,、::;;。.!!??  " +#: 分句的标点:叫法截到第一个之前(「销售额,主要来自华东」只留「销售额」) +_CLAUSE = re.compile(r"[,。;:!?、,.;:!?]") +#: 截出来以后还没配上对的左括号(「销售额(含税,元)」截成「销售额(含税」)连同后面的都不要 +_OPENERS = {"(": ")", "(": ")", "【": "】"} def clean_label(text: Any) -> str: - """消毒(见文件头):去引用标记、数字(出具校验认的数字写法连同后缀,再加剩下的数字字符)、百分号和单位符号、空括号, - 压掉多余空白,不超过 LABEL_MAX 字。可能消成空串。""" + """消毒(见文件头):去引用标记、数字(出具校验认的数字写法连同后缀,再加剩下的数字字符)、百分号和单位符号、空括号、 + 多余空白,反复做到不再变化(去掉一处可能拼出新的一处:「销售额[ [v:Q1.r0.gmv] ]」去完数字成了「销售额[[v:Q.r.gmv]]」), + 再去掉剩下的方括号,截到第一个分句标点之前、去掉没配上对的左括号,不超过 LABEL_MAX 字;整个过程再做到不再变化,所以 + 消过的再消一遍不变。消完的叫法里有说法表的判断词(engine/analysis_claims.scan 认得出的)就整个不要。可能消成空串。""" if not isinstance(text, str): return "" + out = text + for _ in range(len(text) + 2): + again = _shorten(_settle(out)) + if again == out: + break + out = again + return "" if out and _judges(out) else out + + +def _settle(text: str) -> str: + """去引用标记、数字、符号、空括号,做到不再变化(每一遍只会变短或只换空白,一定停得下来)。""" + out = text + for _ in range(len(text) + 2): + again = _scrub(out) + if again == out: + break + out = again + return out + + +def _shorten(text: str) -> str: + """去掉剩下的方括号,截到第一个分句标点之前,去掉没配上对的左括号连同后面的,不超过 LABEL_MAX 字。""" + out = _CLAUSE.split(text.replace("[", "").replace("]", ""), maxsplit=1)[0] + open_at: list[tuple[int, str]] = [] + for i, ch in enumerate(out): + if ch in _OPENERS: + open_at.append((i, _OPENERS[ch])) + elif open_at and ch == open_at[-1][1]: + open_at.pop() + if open_at: + out = out[:open_at[0][0]] + out = re.sub(r"\s+", " ", out).strip(_EDGE_PUNCT) + return out[:LABEL_MAX].strip(_EDGE_PUNCT) + + +def _judges(text: str) -> bool: + """叫法里有没有说法表的判断词(「主要来自」「历史新高」……)。说法核对在 engine 里,在这里导入免得循环导入。""" + from app.engine.analysis_claims import scan + + return bool(scan(text)) + + +def _scrub(text: str) -> str: + """一遍:去引用标记、数字、符号,收拾去掉以后留下的空白、标点、空括号。""" from app.engine.issuance import extract_numbers raw = _MARKER.sub("", text) @@ -193,8 +245,7 @@ def clean_label(text: Any) -> str: out = re.sub(r"([,,、::;;])[,,、::;;]+", r"\1", out) while _EMPTY_BRACKETS.search(out): out = _EMPTY_BRACKETS.sub("", out) - out = re.sub(r"\s+", " ", out).strip(_EDGE_PUNCT) - return out[:LABEL_MAX].strip(_EDGE_PUNCT) + return re.sub(r"\s+", " ", out).strip(_EDGE_PUNCT) def pick_label(*candidates: Any, fallback: str = "这一列") -> str: diff --git a/backend/tests/test_analysis_decompose.py b/backend/tests/test_analysis_decompose.py index dbc31725..9710e192 100644 --- a/backend/tests/test_analysis_decompose.py +++ b/backend/tests/test_analysis_decompose.py @@ -448,3 +448,16 @@ async def test_dimension_columns_keep_the_type_recorded_at_query_time(): factor_result = await factor([["W41", "1000", 100, "10"], ["W42", "1320", 120, "11"]]) assert factor_result["table"]["column_types"] == {"factor": "text", "base": "number", "target": "number", "pct": "number", "contrib": "number", "share": "number"} + + + +async def test_a_label_cannot_carry_a_judgment_into_the_phrase(): + cut = await additive(label="销售额,主要来自华南") + assert cut["params"]["label"] == "销售额" and cut["phrase"].startswith("销售额下降 194") + assert "华南" not in cut["phrase"] + judged = await additive(label="主要来自华南的销售额") + assert "label" not in judged["params"] # 消成空了,退回目录里的叫法 + assert judged["phrase"].startswith("销售额下降 194") and "华南" not in judged["phrase"] + # 消过一遍会变的叫法:复算拿记下的叫法再消,判定语逐字相同(analyze 核对复算逐字段相同) + marked = await additive(label="销售额[ [v:Q1.r0.gmv] ]") + assert marked["params"]["label"] == "销售额" and marked["phrase"].startswith("销售额下降 194") diff --git a/backend/tests/test_analysis_phrases.py b/backend/tests/test_analysis_phrases.py index 147d53d8..108becf1 100644 --- a/backend/tests/test_analysis_phrases.py +++ b/backend/tests/test_analysis_phrases.py @@ -60,6 +60,28 @@ def test_labels_are_scrubbed_of_numbers_units_and_markers(): assert ph.pick_label("2026", None, "2026") == "这一列" +def test_scrubbing_a_scrubbed_label_changes_nothing(): + # 去掉一处可能拼出新的一处:去完数字的「销售额[[v:Q.r.gmv]]」又是一个引用标记,要接着去 + assert ph.clean_label("销售额[ [v:Q1.r0.gmv] ]") == "销售额" + assert ph.clean_label("[ [see:A1] ]销售额") == "销售额" and ph.clean_label("销售额[1[v:x]2]") == "销售额" + for text in ("销售额[ [v:Q1.r0.gmv] ]", "销售额 [ [ a:A1 ] ]", "销售额( )(3)", "销售额(同比 +30%,3.5万)", + "近三成客户的退款率", "【线上】销售额", "退款率 (%)", "销售额(", " ,销售额。", "一[二]三", + "销售额" * 30, "[[a:A1]]主要来自[[v:Q1.r0.region]]"): + once = ph.clean_label(text) + assert ph.clean_label(once) == once, text + + +def test_labels_stop_at_the_first_clause_and_judgments_fall_back(): + assert ph.clean_label("销售额,主要来自华东") == "销售额" + assert ph.clean_label("订单数; 销售额") == "订单数" and ph.clean_label("订单数、销售额") == "订单数" + assert ph.clean_label("销售额(含税,元)") == "销售额" # 截完没配上对的括号也不要 + # 判定语不受说法核对:叫法里有判断词的整个不要,退回下一个来源(目录叫法、当叫法用的列名同样处理) + assert ph.clean_label("销售额主要来自华东") == "" and ph.clean_label("历史新高的销售额") == "" + assert ph.pick_label("销售额主要来自华东", "实收金额", "gmv") == "实收金额" + assert ph.pick_label(None, "历史新高的销售额", "gmv") == "gmv" + assert ph.pick_label(None, None, "主要来自华东") == "这一列" + + def test_numbers_stuffed_into_a_label_never_reach_the_phrase(): label = ph.pick_label("销售额(同比 +30%,3.5万)", "销售额", "gmv") phrase = ph.decompose_driver(label, ph.change(-194, 2220), "华东", "-120", Fraction(6186, 10000)) From 6e38df4fbfef48ce8581e49d9a1584f86773c5da Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 15:47:30 +0800 Subject: [PATCH 03/13] =?UTF-8?q?=E5=88=86=E6=9E=90=E5=8D=A1=E9=87=8C?= =?UTF-8?q?=E9=9A=8F=E6=95=B0=E6=8D=AE=E5=8F=98=E5=8C=96=E7=9A=84=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E9=94=99=E8=AF=AF=E8=AE=B0=E6=88=90=E7=AE=97=E4=B8=8D?= =?UTF-8?q?=E4=BA=86=E7=9A=84=E4=B8=80=E9=A1=B9=EF=BC=8C=E4=B8=8D=E8=AE=A9?= =?UTF-8?q?=E5=8D=A1=E5=A4=B1=E8=B4=A5=EF=BC=88=E5=A4=8D=E6=A0=B8=20R6?= =?UTF-8?q?=EF=BC=89=EF=BC=9Aanalysis.run=20=E5=8A=A0=E5=8F=82=E6=95=B0=20?= =?UTF-8?q?refuse=5Fdata=5Ferrors=EF=BC=88=E7=BC=BA=E7=9C=81=E5=81=87?= =?UTF-8?q?=EF=BC=89=EF=BC=8C=E4=B8=BA=E7=9C=9F=E6=97=B6=20DATA=5FERRORS?= =?UTF-8?q?=20=E9=87=8C=E7=9A=84=E8=B0=83=E7=94=A8=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E2=80=94=E2=80=94=E4=B8=8A=E4=B8=80=E5=B1=82=E6=B2=A1=E5=81=9A?= =?UTF-8?q?=E6=88=90=20parent=5Frefused=E3=80=81=E4=B8=8B=E9=92=BB?= =?UTF-8?q?=E7=9A=84=E6=AE=B5=E8=BF=99=E6=9C=9F=E6=B2=A1=E6=9C=89=20parent?= =?UTF-8?q?=5Fkey=5Fmissing=E3=80=81=E6=9C=9F=E6=AC=A1=E8=BF=99=E6=9C=9F?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=20period=5Fnot=5Ffound=EF=BC=88=E5=8F=98?= =?UTF-8?q?=E5=8C=96=E6=8B=86=E8=A7=A3=E3=80=81=E6=B3=A2=E5=8A=A8=E5=9F=BA?= =?UTF-8?q?=E7=BA=BF=E7=9A=84=E6=9C=AC=E6=9C=9F=E3=80=81=E5=B7=AE=E5=BC=82?= =?UTF-8?q?=E6=A3=80=E9=AA=8C=E6=8C=89=E6=9C=9F=E6=AF=94=E3=80=81=E5=8F=AF?= =?UTF-8?q?=E6=AF=94=E6=80=A7=E6=A3=80=E6=9F=A5=EF=BC=89=E3=80=81=E6=80=BB?= =?UTF-8?q?=E6=95=B0=E9=82=A3=E4=B8=80=E6=A0=BC=E8=B6=8A=E7=95=8C=E6=88=96?= =?UTF-8?q?=E4=B8=BA=E7=A9=BA=20row=5Fout=5Fof=5Frange=20/=20headline=5Fem?= =?UTF-8?q?pty=E3=80=81=E8=A6=81=E6=AF=94=E7=9A=84=E7=BB=84=E8=BF=99?= =?UTF-8?q?=E6=9C=9F=E6=B2=A1=E6=9C=89=20unknown=5Fgroup=20/=20too=5Ffew?= =?UTF-8?q?=5Fgroups=E3=80=81=E8=A6=81=E7=94=A8=E7=9A=84=E9=82=A3=E4=B8=80?= =?UTF-8?q?=E6=A0=BC=E8=BF=99=E6=9C=9F=E4=B8=BA=E7=A9=BA=20empty=5Fcell?= =?UTF-8?q?=EF=BC=88=E4=B9=98=E7=A7=AF=E6=8B=86=E8=A7=A3=E7=9A=84=E4=B9=98?= =?UTF-8?q?=E7=A7=AF=E5=92=8C=E5=9B=A0=E5=AD=90=EF=BC=9B=E5=B7=AE=E5=BC=82?= =?UTF-8?q?=E6=A3=80=E9=AA=8C=E7=9A=84=E8=AE=A1=E6=95=B0=E4=B8=BA=E7=A9=BA?= =?UTF-8?q?=E5=8E=9F=E6=9D=A5=E6=8A=A5=E7=9A=84=E6=98=AF=20not=5Fcount?= =?UTF-8?q?=EF=BC=8C=E6=94=B9=E6=8A=A5=20empty=5Fcell=EF=BC=89=E2=80=94?= =?UTF-8?q?=E2=80=94=E8=AE=B0=E6=88=90=20status=20refused=20=E7=9A=84?= =?UTF-8?q?=E7=BB=93=E8=AE=BA=EF=BC=9A=E5=A5=91=E7=BA=A6=20REASONS=20?= =?UTF-8?q?=E5=8A=A0=20parent=5Frefused=E3=80=81input=5Funavailable?= =?UTF-8?q?=EF=BC=8CCallError=20=E5=8A=A0=20reason=EF=BC=88=E7=85=A7?= =?UTF-8?q?=E5=AE=9E=E8=AF=B4=E6=98=AF=E5=93=AA=E4=B8=80=E9=A1=B9=E3=80=81?= =?UTF-8?q?=E5=93=AA=E4=B8=AA=E5=80=BC=EF=BC=8C=E4=B8=8D=E5=B8=A6=E6=94=B9?= =?UTF-8?q?=E6=B3=95=EF=BC=89=EF=BC=8C=E5=88=A4=E5=AE=9A=E8=AF=AD=E3=80=8C?= =?UTF-8?q?{=E5=8F=AB=E6=B3=95}=E6=B2=A1=E6=9C=89=E5=81=9A=E6=88=90?= =?UTF-8?q?=EF=BC=9A=E5=8E=9F=E5=9B=A0=E3=80=8D=EF=BC=9Bparams=20=E7=85=A7?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E5=8E=9F=E6=A0=B7=EF=BC=88label=20=E7=85=A7?= =?UTF-8?q?=E6=A0=B7=E6=B6=88=E6=AF=92=EF=BC=89=EF=BC=8C=E5=8F=96=E8=BF=87?= =?UTF-8?q?=E8=BF=98=E6=B2=A1=E8=AE=B0=E7=94=A8=E6=B3=95=E7=9A=84=E5=BF=AB?= =?UTF-8?q?=E7=85=A7=E9=83=BD=E8=AE=B0=E8=BF=9B=E8=BE=93=E5=85=A5=EF=BC=88?= =?UTF-8?q?Inputs.settle=EF=BC=8C=E8=A7=92=E8=89=B2=E7=85=A7=E7=AC=AC?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E5=8F=96=E6=97=B6=E7=9A=84=EF=BC=89=EF=BC=8C?= =?UTF-8?q?=E4=B8=8A=E4=B8=80=E5=B1=82=E5=9C=A8=E6=A0=B8=E5=AF=B9=E5=AE=83?= =?UTF-8?q?=E5=81=9A=E6=B2=A1=E5=81=9A=E6=88=90=E4=B9=8B=E5=89=8D=E5=B0=B1?= =?UTF-8?q?=E8=AE=B0=E8=BF=9B=E8=BE=93=E5=85=A5=EF=BC=8C=E5=A4=8D=E7=AE=97?= =?UTF-8?q?=E6=8C=89=E5=AE=83=E4=BB=AC=E8=B5=B0=E5=88=B0=E5=90=8C=E4=B8=80?= =?UTF-8?q?=E5=A4=84=EF=BC=9B=E9=98=88=E5=80=BC=E5=92=8C=E5=88=A4=E5=AE=9A?= =?UTF-8?q?=E6=A0=87=E5=87=86=E5=90=8C=E5=90=84=E7=AE=97=E5=AD=90=E8=87=AA?= =?UTF-8?q?=E5=B7=B1=E7=AE=97=E4=B8=8D=E4=BA=86=E6=97=B6=E9=82=A3=E4=B8=80?= =?UTF-8?q?=E4=BB=BD=EF=BC=88=E5=90=84=E6=A8=A1=E5=9D=97=E7=9A=84=20USED?= =?UTF-8?q?=20/=20refusal=5Fbasis=EF=BC=8C=E7=AE=97=E5=AD=90=E6=9C=AC?= =?UTF-8?q?=E8=BA=AB=E4=B9=9F=E6=94=B9=E7=94=A8=E5=AE=83=EF=BC=89=E3=80=82?= =?UTF-8?q?recompute=20=E4=B8=80=E5=BE=8B=E6=8C=89=E7=9C=9F=E5=A4=8D?= =?UTF-8?q?=E7=8E=B0=EF=BC=9B=E7=BB=93=E6=9E=84=E6=80=A7=E7=9A=84=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E9=94=99=E8=AF=AF=E7=85=A7=E6=97=A7=E6=8A=9B=E3=80=82?= =?UTF-8?q?=E5=88=86=E6=9E=90=E5=8D=A1=E8=BF=90=E8=A1=8C=E5=99=A8=E4=BC=A0?= =?UTF-8?q?=E7=9C=9F=EF=BC=8Cagent=20=E5=B7=A5=E5=85=B7=E4=B8=8D=E4=BC=A0?= =?UTF-8?q?=E3=80=82=E9=A1=BA=E6=89=8B=EF=BC=9A=E6=AE=B5=E6=95=B0=E4=B8=8D?= =?UTF-8?q?=E5=A4=9F=E7=9A=84=E5=8E=9F=E5=9B=A0=E9=87=8C=E7=BB=B4=E5=BA=A6?= =?UTF-8?q?=E5=8F=AB=E6=B3=95=E5=92=8C=E6=B1=89=E5=AD=97=E4=B9=8B=E9=97=B4?= =?UTF-8?q?=E7=85=A7=20say=20=E7=9A=84=E8=A7=84=E7=9F=A9=E8=A1=A5=E7=A9=BA?= =?UTF-8?q?=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/__init__.py | 62 ++++++++++++++-- backend/app/analysis/comparable.py | 8 +- backend/app/analysis/contract.py | 14 +++- backend/app/analysis/decompose.py | 56 ++++++++++---- backend/app/analysis/differ.py | 28 +++++-- backend/app/analysis/inputs.py | 30 ++++++-- backend/app/analysis/periods.py | 3 +- backend/app/analysis/series.py | 20 ++++- backend/app/engine/nodes/analysis.py | 10 ++- backend/tests/analysis_samples.py | 8 +- backend/tests/fixtures/analysis_ops.py | 4 +- backend/tests/test_analysis_card_data.py | 70 ++++++++++++++++++ backend/tests/test_analysis_run.py | 94 ++++++++++++++++++++++++ 13 files changed, 353 insertions(+), 54 deletions(-) create mode 100644 backend/tests/test_analysis_card_data.py diff --git a/backend/app/analysis/__init__.py b/backend/app/analysis/__init__.py index 314806da..ecdb0c02 100644 --- a/backend/app/analysis/__init__.py +++ b/backend/app/analysis/__init__.py @@ -5,8 +5,11 @@ - run:按请求算一份分析结论。resolve 按调用里写的名字(Q3、来源节点 id)取回输入快照;单元格写成「名字.r行.列」 (Q1.r0.gmv),由这里拆开再经 resolve 取快照。parents 按名字(A3、分析卡里的项 id)取回上一层拆解,下钻用。 调用写错抛 contract.CallError;算不了返回 status refused 的结论。唯一的异步来自 context["probe"](现查表的日期范围)。 + refuse_data_errors 为真时(分析卡:调用是模板定死的,这期的数据里没有要用的那一项不是写错了),随数据变化的调用错误 + (DATA_ERRORS)也记成算不了的结论:上一层没做成 parent_refused,其余 input_unavailable,原因照实写是哪一项、哪个值; + 结构性的(列不存在、遮罩列、比率当可加、重复行、参数写错)照旧抛。agent 工具不传,调用错误交回模型改调用。 - recompute:按结论里记下的 params、settings、probes 和输入快照(按 sha256 取回)重算一遍,不现查;和封存的用 - contract.same 比。 + contract.same 比。一律按 refuse_data_errors 为真复现:分析卡记下的这类结论复算也成立。 契约(名字、形状、缺省阈值)在 contract.py。各算子在 decompose.py(变化拆解)、series.py(波动基线、趋势)、differ.py (差异检验)、comparable.py(可比性检查);统计在 stats.py,判定语和数的写法在 phrases.py,现查在 probe.py。 @@ -32,6 +35,14 @@ Resolver = Callable[[str], InputSnapshot | None] #: 按名字取回上一层的分析结论(下钻);没有返回 None ParentResolver = Callable[[str], AnalysisResult | None] +#: 随数据变化的调用错误(CallError.code):调用本身没写错,是这期的数据里没有要用的那一项。分析卡里(refuse_data_errors) +#: 记成算不了的结论,不让卡失败:上一层没做成(parent_refused);下钻的段这期没有(parent_key_missing);期次这期没有 +#: (period_not_found:变化拆解、波动基线的本期、差异检验按期比、可比性检查);按位置取的总数那一格越界或为空 +#: (row_out_of_range、headline_empty);要比的组这期没有(unknown_group、too_few_groups);要用的那一格这期为空 +#: (empty_cell:乘积拆解的乘积、因子,差异检验的计数) +DATA_ERRORS = frozenset({"parent_refused", "parent_key_missing", "period_not_found", "row_out_of_range", + "headline_empty", "unknown_group", "too_few_groups", "empty_cell"}) + def _op_mode(request: Mapping[str, Any]) -> tuple[str, str | None]: op = request.get("op") @@ -69,9 +80,44 @@ def _env(op: str, mode: str | None, settings: Mapping[str, Any], inputs: Any, fa parents=parents, hypothesis=hypothesis if isinstance(hypothesis, str) else None) +def _data_refusal(op: str, mode: str | None, raw: Mapping[str, Any], env: Any, error: CallError) -> AnalysisResult: + """随数据变化的调用错误记成的算不了的结论。params 记调用原样(label 照样消毒;调用还没走完、规范化不完整,复算拿它重走一遍 + 走到同一处),取过的快照都记进输入(Inputs.settle),阈值和判定标准同这个算子自己算不了时那一份。""" + from app.analysis import decompose, differ, series + from app.analysis import phrases as ph + from app.analysis.common import build + + env.inputs.settle() + params = dict(raw) + if params.get("label") is not None: + params["label"] = ph.clean_label(params["label"]) or None + if op == "decompose": + used, criterion = decompose.refusal_basis(mode, raw, env.settings) # type: ignore[arg-type] + elif op in ("baseline", "trend"): + used, criterion = series.refusal_basis(op, env.settings) + elif op == "differ": + used, criterion = differ.refusal_basis(mode, env.settings) + else: + used, criterion = [], ph.CRITERION_COMPARABLE + code = "parent_refused" if error.code == "parent_refused" else "input_unavailable" + reason = error.reason or error.message + return build(env, params, used, status="refused", reason_code=code, reason=reason, criterion=criterion, + phrase=ph.refused(op, reason)) + + +def _compute_or_refuse(op: str, mode: str | None, raw: Mapping[str, Any], env: Any, refuse_data_errors: bool + ) -> AnalysisResult: + try: + return _compute(op, mode, raw, env) + except CallError as e: + if refuse_data_errors and e.code in DATA_ERRORS: + return _data_refusal(op, mode, raw, env, e) + raise + + async def run(request: AnalysisRequest, resolve: Resolver, *, settings: Mapping[str, Any], context: Context, - parents: ParentResolver | None = None) -> AnalysisResult: - """算一份分析结论。settings 是 contract.effective_settings 算好的阈值。""" + parents: ParentResolver | None = None, refuse_data_errors: bool = False) -> AnalysisResult: + """算一份分析结论。settings 是 contract.effective_settings 算好的阈值。refuse_data_errors 见文件头(分析卡传真)。""" from app.analysis.common import DERIVED_PARAMS from app.analysis.facts import Facts from app.analysis.inputs import Inputs @@ -97,7 +143,7 @@ async def run(request: AnalysisRequest, resolve: Resolver, *, settings: Mapping[ book = Book(known) env = _env(op, mode, full, Inputs(resolve, dialect), Facts(catalogs=context.get("catalogs")), book, today, parents, request.get("hypothesis")) - result = _compute(op, mode, raw, env) + result = _compute_or_refuse(op, mode, raw, env, refuse_data_errors) missing = book.missing if not missing or rounds == MAX_ROUNDS: return result @@ -109,7 +155,8 @@ def recompute(result: AnalysisResult, load: Callable[[str], InputSnapshot | None load_result: Callable[[str], AnalysisResult | None] | None = None) -> AnalysisResult: """复算:load 按 sha256 取回输入快照,load_result 按 sha256 取回 parent 那份结论。用结论里记下的 params(含运行当天、 目录事实)、settings、probes,不现查、不读目录。取不回输入时抛 CallError(code input_missing);方法升版后调用不再成立 - 时同样抛 CallError(照 run 的说法)。和封存的那份用 contract.same 比。""" + 时同样抛 CallError(照 run 的说法)。随数据变化的调用错误一律记成算不了(同 run 的 refuse_data_errors 为真),分析卡记下 + 的这类结论复算也成立。和封存的那份用 contract.same 比。""" from app.analysis.common import DERIVED_PARAMS from app.analysis.facts import Facts from app.analysis.inputs import _CELL, Inputs @@ -150,7 +197,7 @@ def recompute(result: AnalysisResult, load: Callable[[str], InputSnapshot | None env = _env(op, mode, settings, Inputs(resolve, dialect), Facts(recorded=params.get("catalog") or {}), Book(result.get("probes") or []), today, parents, result.get("hypothesis")) raw = {k: v for k, v in params.items() if k not in DERIVED_PARAMS} - return _compute(op, mode, raw, env) + return _compute_or_refuse(op, mode, raw, env, True) def exhausted(settings: Mapping[str, Any]) -> CallError: @@ -174,4 +221,5 @@ def comparable_reasons(result: Mapping[str, Any]) -> list[str]: return reasons(result) -__all__ = ["ParentResolver", "Resolver", "comparable_reasons", "exhausted", "make_prober", "recompute", "run"] +__all__ = ["DATA_ERRORS", "ParentResolver", "Resolver", "comparable_reasons", "exhausted", "make_prober", "recompute", + "run"] diff --git a/backend/app/analysis/comparable.py b/backend/app/analysis/comparable.py index 3990222e..76041bb6 100644 --- a/backend/app/analysis/comparable.py +++ b/backend/app/analysis/comparable.py @@ -497,8 +497,9 @@ def comparable(raw: Mapping[str, Any], env: Any) -> Any: params: dict[str, Any] = {} two = raw.get("base_query") is not None or raw.get("target_query") is not None if two: - base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号") - target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号") + base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号", role="base") + target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号", + role="target") if base is target: from app.analysis.decompose import SAME_QUERY @@ -529,7 +530,8 @@ def comparable(raw: Mapping[str, Any], env: Any) -> Any: for side, what in (("base", "基期"), ("target", "本期")): if not rows[side]: raise CallError("period_not_found", f"期次列「{period}」里没有「{params[side]}」:{what}要写 {target.ref} " - "里有的期次值") + "里有的期次值", + reason=f"期次列「{period}」里没有{what}「{ph.key_text([params[side]])}」") env.inputs.use(target, "data", [period, *([measure] if measure else [])], rows["base"] + rows["target"]) parsed = pd.parse(values, grain, weeks=pd.week_scheme(target.sql, period, target.dialect)) if parsed is None: diff --git a/backend/app/analysis/contract.py b/backend/app/analysis/contract.py index eb038afd..965e1bbb 100644 --- a/backend/app/analysis/contract.py +++ b/backend/app/analysis/contract.py @@ -83,10 +83,12 @@ VERDICTS: dict[str, tuple[str, ...]] = { "differ": ("differs", "no_evidence"), "comparable": ("comparable", "conditional", "incomparable"), } -#: 算不了的原因(细案 01 §8.2)。算不了也记成结论:写作者可以照实写「历史只有 5 周,判断不了是否异常」 +#: 算不了的原因(细案 01 §8.2)。算不了也记成结论:写作者可以照实写「历史只有 5 周,判断不了是否异常」。最后两种是分析卡 +#: 里随数据变化的调用错误记成的(analysis.run 的 refuse_data_errors,见 app/analysis 的 DATA_ERRORS):parent_refused 上一层 +#: 没做成;input_unavailable 这次的数据里没有要用的那一项(下钻的段、期次、总数那一格、要比的组……,reason 写是哪一项、哪个值) REASONS: tuple[str, ...] = ( "truncated", "too_few_segments", "history_short", "partial_target", "target_gap", "series_gap", - "period_unreadable", "zero_base", "non_positive", "incomparable_sides", + "period_unreadable", "zero_base", "non_positive", "incomparable_sides", "parent_refused", "input_unavailable", ) #: 附带说明的种类(caveats[].code) CAVEATS: tuple[str, ...] = ( @@ -305,12 +307,16 @@ class AnalysisResult(TypedDict, total=False): class CallError(Exception): """调用错误(细案 01 §8.1):列不存在、列被遮罩、角色和目录度量类型冲突、同一键同一期多行、期次不在列里、乘积 不闭合、parent 不对、下钻超深、这个 Agent 这次执行的分析次数用完(max_calls 按每个 Agent 每次执行计)。不记结论,message - 交回 agent(一种说法,告诉它怎么改)。""" + 交回 agent(一种说法,告诉它怎么改)。 - def __init__(self, code: str, message: str) -> None: + 随数据变化的那几种(app/analysis 的 DATA_ERRORS:这期没有要用的期次、段、组,上一层没做成……)另带 reason:照实说是哪一项、 + 哪个值,不带改法。分析卡(模板定死了调用)按它记成算不了的结论,见 analysis.run 的 refuse_data_errors。""" + + def __init__(self, code: str, message: str, *, reason: str | None = None) -> None: super().__init__(message) self.code = code self.message = message + self.reason = reason #: 现查一列的实际范围:(数据源名, 表, 列) → Probe;查不了返回 None(算子写「核对不了」,不判不可比) diff --git a/backend/app/analysis/decompose.py b/backend/app/analysis/decompose.py index 0079d2a1..1d42bb88 100644 --- a/backend/app/analysis/decompose.py +++ b/backend/app/analysis/decompose.py @@ -41,6 +41,22 @@ SAME_QUERY = "基期和本期是同一份查询:两期各给一份查询,或 #: 乘积闭合的最小相对误差 _FACTOR_REL = Fraction(1, 10 ** 6) _NOT_ADDITIVE = {"ratio": "比率", "identifier": "标识", "status": "状态", "attribute": "属性"} +#: 各模式的结论里记的阈值(下钻时另加 max_drill_depth) +USED: dict[str, tuple[str, ...]] = { + "additive": ("main_share", "concentration", "offset_ratio", "flat_pct", "table_top_k"), + "ratio": ("main_share", "flat_pct", "min_denominator", "table_top_k"), + "factor": ("main_share", "flat_pct"), +} + + +def refusal_basis(mode: str, raw: Mapping[str, Any], settings: Mapping[str, Any]) -> tuple[list[str], str]: + """还没算就算不了时(分析卡把随数据变化的调用错误记成算不了的结论)结论里记的阈值和判定标准:同这个模式自己算不了时 + 的那一份。""" + used = list(USED[mode]) + (["max_drill_depth"] if mode != "factor" and raw.get("parent") is not None else []) + if mode == "additive": + summed = raw.get("headline") is None and raw.get("parent") is None + return used, ph.criterion_additive(settings, top_k_max=None, summed=summed) + return used, ph.criterion_ratio(settings) if mode == "ratio" else ph.criterion_factor(settings) @dataclass @@ -71,8 +87,9 @@ def _sides(raw: Mapping[str, Any], env: Env, roles: Sequence[tuple[str, str]], * params: dict[str, Any] = {} two = raw.get("base_query") is not None or raw.get("target_query") is not None if two: - base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号") - target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号") + base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号", role="base") + target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号", + role="target") if base is target: raise CallError("same_query", SAME_QUERY) params.update(base_query=base.ref, target_query=target.ref) @@ -116,7 +133,8 @@ def _period_rows(snap: Snap, period: str, wanted: Any, what: str) -> list[int]: if not rows: values = list(dict.fromkeys(ph.key_text([row[i]]) for row in snap.rows if i < len(row))) raise CallError("period_not_found", f"期次列「{period}」里没有「{wanted}」:{what}要写 {snap.ref} 里有的期次值," - f"比如 {'、'.join(values[:6])}") + f"比如 {'、'.join(values[:6])}", + reason=f"期次列「{period}」里没有{what}「{ph.key_text([wanted])}」") return rows @@ -181,8 +199,12 @@ def _parent(raw: Mapping[str, Any], env: Env, mode: str, dims: int) -> tuple[dic if found.get("op") != "decompose" or found.get("mode") != mode: raise CallError("parent_not_decompose", f"{name} 不是{'按维度' if mode == 'additive' else '比率'}的变化拆解," "下钻要接在同一种拆解后面") + # 上一层先记进输入:它没做成、这期没有那一段时(分析卡记成算不了的结论),复算也要按内容哈希取回它、走到同一处 + env.inputs.parent(name, content_hash(canonical_json(found))) if found.get("status") != "ok" or not isinstance(found.get("table"), Mapping): - raise CallError("parent_refused", f"{name} 没有做成,不能在它下面下钻") + why = f"({found['reason']})" if isinstance(found.get("reason"), str) and found["reason"] else "" + raise CallError("parent_refused", f"{name} 没有做成,不能在它下面下钻", + reason=f"上一层「{name}」没有做成{why},不能在它下面下钻") depth = int((found.get("params") or {}).get("depth") or 0) + 1 limit = int(env.settings["max_drill_depth"]) if depth > limit: @@ -196,8 +218,8 @@ def _parent(raw: Mapping[str, Any], env: Env, mode: str, dims: int) -> tuple[dic hit = next((row for row in table.get("rows") or [] if all(same_value(k, v) for k, v in zip(key, row[:dim_count]))), None) if hit is None: - raise CallError("parent_key_missing", f"{name} 里没有「{ph.key_text(key)}」这一段:写 {name} 结果表里的段") - env.inputs.parent(name, content_hash(canonical_json(found))) + raise CallError("parent_key_missing", f"{name} 里没有「{ph.key_text(key)}」这一段:写 {name} 结果表里的段", + reason=f"上一层「{name}」里没有「{ph.key_text(key)}」这一段") return {"of": name, "key": key}, dict(zip(columns, hit)), depth @@ -220,7 +242,8 @@ def _headline(raw: Mapping[str, Any], env: Env) -> tuple[dict[str, str] | None, for side, what in (("base", "基期总数"), ("target", "本期总数")): snap, row, col, value = env.inputs.cell(spec[side], f"headline_{side}", what) if value is None: - raise CallError("headline_empty", f"{what}那一格({snap.ref}.r{row}.{col})是空的:换一格有值的") + raise CallError("headline_empty", f"{what}那一格({snap.ref}.r{row}.{col})是空的:换一格有值的", + reason=f"{what}那一格({snap.ref}.r{row}.{col})是空的") out[side] = (value, snap.cell(row, col)) refs[side] = f"{snap.ref}.r{row}.{col}" return refs, out @@ -260,7 +283,7 @@ def _share_rank(segments: Sequence[Any], share: Any) -> list[Any]: def additive(raw: Mapping[str, Any], env: Env) -> Any: - used = ["main_share", "concentration", "offset_ratio", "flat_pct", "table_top_k"] + used = list(USED["additive"]) params, bs, ts, cols, by = _sides(raw, env, [("measure", "度量列")]) measure = cols["measure"] label = label_for(env, raw.get("label"), ts, measure) @@ -299,7 +322,8 @@ def additive(raw: Mapping[str, Any], env: Env) -> Any: criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats) segs = _segments(side_b, side_t, Fraction(0)) if len(segs) < 2: - return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(segs)} 段,至少要 2 段才拆得开", + return refuse(env, params, used, "too_few_segments", + ph.say("按", ph.dims_text(dims), f"只有 {len(segs)} 段,至少要 2 段才拆得开"), criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats) return _additive_result(env, params, used, segs, by, dims, label, heads, parent_row, outcome, caveats, summed, side_b, side_t) @@ -419,7 +443,7 @@ def _additive_result(env: Env, params: dict[str, Any], used: list[str], segs: li def ratio(raw: Mapping[str, Any], env: Env) -> Any: - used = ["main_share", "flat_pct", "min_denominator", "table_top_k"] + used = list(USED["ratio"]) params, bs, ts, cols, by = _sides(raw, env, [("numerator", "分子列"), ("denominator", "分母列")]) num, den = cols["numerator"], cols["denominator"] num_label, den_label = column_label(env, ts, num), column_label(env, ts, den) @@ -462,7 +486,8 @@ def ratio(raw: Mapping[str, Any], env: Env) -> Any: segs = _segments(side_b, side_t, (Fraction(0), Fraction(0))) live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0] if len(live) < 2: - return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(live)} 段,至少要 2 段才拆得开", + return refuse(env, params, used, "too_few_segments", + ph.say("按", ph.dims_text(dims), f"只有 {len(live)} 段,至少要 2 段才拆得开"), criterion=criterion) for when, i in (("基期", 0), ("本期", 1)): if sum(((x.base, x.target)[i][1] for x in live), Fraction(0)) == 0: @@ -635,7 +660,7 @@ def lmdi(base: Sequence[float], target: Sequence[float], m_base: float, m_target def factor(raw: Mapping[str, Any], env: Env) -> Any: - used = ["main_share", "flat_pct"] + used = list(USED["factor"]) params, bs, ts, cols, _ = _sides(raw, env, [("product", "乘积列")], by_needed=False) product = cols["product"] wanted = as_list(raw.get("factors")) @@ -672,7 +697,8 @@ def factor(raw: Mapping[str, Any], env: Env) -> Any: for col, what in ((product, "乘积列"), *((c, "因子列") for c in factors)): value = snap.number(r, col, what) if value is None: - raise CallError("empty_cell", f"{snap.ref} 第 {r} 行的「{col}」是空的:乘积、因子都要有值") + raise CallError("empty_cell", f"{snap.ref} 第 {r} 行的「{col}」是空的:乘积、因子都要有值", + reason=f"{'基期' if when == 'base' else '本期'}的「{col}」那一格是空的({snap.ref} 第 {r} 行)") row.append((value, snap.cell(r, col))) cells[when] = row if bs is ts: @@ -736,5 +762,5 @@ def factor(raw: Mapping[str, Any], env: Env) -> Any: phrase=ph.with_caveats(main, caveats), caveats=caveats) -__all__ = ["HEADLINE_SHAPE", "PARENT_SHAPE", "SAME_QUERY", "SIMPSON_SHARE", "additive", "factor", "lmdi", "ratio", - "ratio_effects"] +__all__ = ["HEADLINE_SHAPE", "PARENT_SHAPE", "SAME_QUERY", "SIMPSON_SHARE", "USED", "additive", "factor", "lmdi", + "ratio", "ratio_effects", "refusal_basis"] diff --git a/backend/app/analysis/differ.py b/backend/app/analysis/differ.py index e8f839af..12e6816b 100644 --- a/backend/app/analysis/differ.py +++ b/backend/app/analysis/differ.py @@ -36,6 +36,16 @@ from app.analysis.numbers import canon, to_fraction TESTS = {"z": "z 检验", "fisher": "Fisher 精确检验", "welch": "Welch t 检验"} +def used_settings(mode: str | None) -> list[str]: + """结论里记的阈值:比率看最小期望频数,均值、逐行看最小样本数。""" + return ["differ_alpha", "min_expected"] if mode == "rate" else ["differ_alpha", "min_n"] + + +def refusal_basis(mode: str | None, settings: Mapping[str, Any]) -> tuple[list[str], str]: + """还没算就算不了时结论里记的阈值和判定标准(同 decompose.refusal_basis)。""" + return used_settings(mode), ph.criterion_differ(settings, mode) # type: ignore[arg-type] + + @dataclass(eq=False) class Group: key: Any @@ -62,7 +72,10 @@ class Pair: def _int(value: Fraction | None, snap: Snap, r: int, column: str) -> int: - if value is None or value.denominator != 1 or value < 0: + if value is None: + raise CallError("empty_cell", f"{snap.ref} 第 {r} 行的「{column}」是空的:计数要有值(在 SQL 里把空值写成 0)", + reason=f"「{column}」那一格是空的({snap.ref} 第 {r} 行)") + if value.denominator != 1 or value < 0: raise CallError("not_count", f"{snap.ref} 第 {r} 行的「{column}」不是不小于 0 的整数:比率比较的分子、分母要是计数") return int(value) @@ -135,7 +148,7 @@ def _verdict(pairs: Sequence[Pair], alpha: Fraction) -> list[Pair]: def differ(raw: Mapping[str, Any], env: Env) -> Any: mode = env.mode - used = ["differ_alpha", "min_expected"] if mode == "rate" else ["differ_alpha", "min_n"] + used = used_settings(mode) snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号") params: dict[str, Any] = {"query": snap.ref} by_period = raw.get("period") is not None @@ -230,7 +243,8 @@ def _groups(snap: Snap, group_col: str, cols: Mapping[str, str], mode: str, by_p if by_period: for w, what in ((params["target"], "本期"), (params["base"], "基期")): if not any(same_value(w, g.key) for g in groups.values()): - raise CallError("period_not_found", f"期次列「{group_col}」里没有「{w}」:{what}要写 {snap.ref} 里有的期次值") + raise CallError("period_not_found", f"期次列「{group_col}」里没有「{w}」:{what}要写 {snap.ref} 里有的期次值", + reason=f"期次列「{group_col}」里没有{what}「{ph.key_text([w])}」") return groups @@ -238,7 +252,8 @@ def _find(groups: Mapping[Any, Group], value: Any, column: str) -> Group: found = next((g for g in groups.values() if same_value(value, g.key)), None) if found is None: shown = "、".join(g.name for g in list(groups.values())[:8]) - raise CallError("unknown_group", f"「{value}」不是「{column}」里的值:要比的组写 {shown} 里的") + raise CallError("unknown_group", f"「{value}」不是「{column}」里的值:要比的组写 {shown} 里的", + reason=f"分组列「{column}」里没有「{ph.key_text([value])}」这一组") return found @@ -261,7 +276,8 @@ def _pairs(raw: Mapping[str, Any], params: dict[str, Any], groups: dict[Any, Gro params["pairs"] = [[p.a.key, p.b.key] for p in out] return out if len(groups) < 2: - raise CallError("too_few_groups", f"「{group_col}」里只有 {len(groups)} 组,至少要两组才比得了") + raise CallError("too_few_groups", f"「{group_col}」里只有 {len(groups)} 组,至少要两组才比得了", + reason=f"分组列「{group_col}」里只有 {len(groups)} 组,至少要两组才比得了") if raw.get("reference") is not None: ref = _find(groups, raw.get("reference"), group_col) params["reference"] = ref.key @@ -372,4 +388,4 @@ def _sensitivity(pairs: list[Pair], mode: str, alpha: Fraction, min_expected: Fr return (value > mean, now) if now != differs else None -__all__ = ["TESTS", "differ"] +__all__ = ["TESTS", "differ", "refusal_basis", "used_settings"] diff --git a/backend/app/analysis/inputs.py b/backend/app/analysis/inputs.py index 88229117..819806f7 100644 --- a/backend/app/analysis/inputs.py +++ b/backend/app/analysis/inputs.py @@ -60,6 +60,7 @@ class Snap: dialect: str | None = None columns: list[str] = field(default_factory=list) rows: list[list[Any]] = field(default_factory=list) + named: list[str] = field(default_factory=list) # 调用里认过的列(按先后,Inputs.settle 记用法用) def __post_init__(self) -> None: self.columns = [str(c) for c in self.data.get("columns") or []] @@ -117,6 +118,8 @@ class Snap: column = hits[0] if column.lower() in self.masked: raise CallError("masked_column", f"「{column}」是遮罩列,不能当{role}:换一列,或者在 SQL 里不选它") + if column not in self.named: + self.named.append(column) return column def index(self, column: str) -> int: @@ -157,18 +160,20 @@ class Inputs: self._resolve = resolve self._dialect = dialect self._snaps: dict[str, Snap] = {} + self._roles: dict[str, str] = {} self.records: list[AnalysisInput] = [] - def snap(self, name: Any, role: str) -> Snap: - """按名字取一份快照。""" + def snap(self, name: Any, what: str, *, role: str = "data") -> Snap: + """按名字取一份快照。what 是调用错误里的叫法;role 是这份快照在结论里的角色(第一次取时的为准,settle 用)。""" if not isinstance(name, str) or not name.strip(): - raise CallError("missing_query", f"没写{role}:写本节点查过的查询编号(如 Q3)") + raise CallError("missing_query", f"没写{what}:写本节点查过的查询编号(如 Q3)") key = name.strip() if key not in self._snaps: found = self._resolve(key) if not isinstance(found, Mapping): - raise CallError("unknown_query", f"没有叫「{key}」的查询结果:{role}写本节点查过的查询编号(如 Q3)") + raise CallError("unknown_query", f"没有叫「{key}」的查询结果:{what}写本节点查过的查询编号(如 Q3)") self._snaps[key] = Snap(key, found, self._dialect(key, found) if self._dialect else found.get("dialect")) + self._roles[key] = role return self._snaps[key] def cell(self, ref: Any, role: str, label: str) -> tuple[Snap, int, str, Fraction | None]: @@ -176,10 +181,11 @@ class Inputs: found = _CELL.match(ref.strip()) if isinstance(ref, str) else None if found is None: raise CallError("bad_cell", f"{label}要写成单元格「编号.r行.列」(如 Q1.r0.gmv),现在是「{ref}」") - snap = self.snap(found.group("name"), label) + snap = self.snap(found.group("name"), label, role=role) row = int(found.group("row")) if row >= len(snap.rows): - raise CallError("row_out_of_range", f"{snap.ref} 没有第 {row} 行(共 {len(snap.rows)} 行,从 r0 数起):改{label}") + raise CallError("row_out_of_range", f"{snap.ref} 没有第 {row} 行(共 {len(snap.rows)} 行,从 r0 数起):改{label}", + reason=f"{label}那一格({ref.strip()})不在:{snap.ref} 这次只有 {len(snap.rows)} 行") column = snap.column(found.group("column"), label) value = snap.number(row, column, label) self.record({"ref": f"{snap.ref}.r{row}.{column}", "role": role, "row": row, "column": column, @@ -209,6 +215,18 @@ class Inputs: if entry not in self.records: self.records.append(entry) # type: ignore[arg-type] + def settle(self) -> None: + """调用在算出结果之前就算不了时(随数据变化的调用错误记成算不了的结论):取过、还没记下用法的快照照样记一条(角色照 + 第一次取时的,列是调用里认过的那几列,行记 all),复算才按名字取得回它们、走到同一处。""" + recorded = set() + for item in self.records: + cell = _CELL.match(str(item.get("ref") or "")) if "row" in item else None + recorded.add(cell.group("name") if cell else item.get("ref")) + for key, snap in self._snaps.items(): + if key not in recorded: + self.record({"ref": key, "role": self._roles.get(key, "data"), "columns": list(snap.named), + "rows": "all"}, snap) + def as_list(value: Any) -> list[Any]: """参数里一个值或一组值:都规范成列表(None 是空列表)。""" diff --git a/backend/app/analysis/periods.py b/backend/app/analysis/periods.py index 5b8a4c9c..8082d248 100644 --- a/backend/app/analysis/periods.py +++ b/backend/app/analysis/periods.py @@ -257,7 +257,8 @@ def week_scheme(sql: str, column: str, dialect: str | None) -> str | None: return None refs = {(name_key(c.table), name_key(c.name)) for c in value.find_all(exp.Column) if not isinstance(c.this, exp.Star)} - texts = [_native_format(node, dialect) for node in value.find_all(exp.TimeToStr, exp.ToChar)] # type: ignore[arg-type] + texts = [_native_format(node, dialect) # type: ignore[arg-type] + for node in value.find_all(exp.TimeToStr, exp.ToChar)] if len(refs) != 1 or not texts or any(t is None for t in texts): return None marks = [m for t in texts for m in _format_marks(t, dialect)] # type: ignore[arg-type] diff --git a/backend/app/analysis/series.py b/backend/app/analysis/series.py index 0a05ac7d..840f4d56 100644 --- a/backend/app/analysis/series.py +++ b/backend/app/analysis/series.py @@ -40,6 +40,17 @@ from app.analysis.numbers import canon, to_fraction MAD_SCALE = Fraction(14826, 10000) #: lag 的上限(周数据同比是 52) _LAG_MAX = 260 +#: 结论里记的阈值 +USED: dict[str, tuple[str, ...]] = { + "baseline": ("k_warn", "k_alert", "min_history", "max_history"), + "trend": ("trend_min_points", "trend_alpha", "max_history"), +} + + +def refusal_basis(op: str, settings: Mapping[str, Any]) -> tuple[list[str], str]: + """还没算就算不了时结论里记的阈值和判定标准(同 decompose.refusal_basis)。""" + criterion = ph.criterion_baseline if op == "baseline" else ph.criterion_trend + return list(USED[op]), criterion(settings, None, "期") @dataclass @@ -147,7 +158,7 @@ def _completeness(env: Env, series: Series, item: pd.Period) -> tuple[str, str | def baseline(raw: Mapping[str, Any], env: Env) -> Any: - used = ["k_warn", "k_alert", "min_history", "max_history"] + used = list(USED["baseline"]) loaded = _load(raw, env, used) if not isinstance(loaded, Series): return loaded @@ -168,7 +179,8 @@ def baseline(raw: Mapping[str, Any], env: Env) -> Any: found = next((p for p in series.points if same_value(raw["target"], p[0].raw)), None) if found is None: raise CallError("period_not_found", f"期次列「{series.period}」里没有「{raw['target']}」(或那一期为空):" - "本期要写序列里有值的一期,不写就是最后一期") + "本期要写序列里有值的一期,不写就是最后一期", + reason=f"期次列「{series.period}」里没有本期「{ph.key_text([raw['target']])}」(或那一期为空)") else: found = series.points[-1] target, x_level = found @@ -277,7 +289,7 @@ def run_length(values: list[Fraction]) -> tuple[int, str | None]: def trend(raw: Mapping[str, Any], env: Env) -> Any: - used = ["trend_min_points", "trend_alpha", "max_history"] + used = list(USED["trend"]) loaded = _load(raw, env, used) if not isinstance(loaded, Series): return loaded @@ -357,4 +369,4 @@ def _tail_gap(series: Series, ordinal: int) -> str: return "再往前缺期" -__all__ = ["MAD_SCALE", "baseline", "run_length", "trend", "unreadable_text"] +__all__ = ["MAD_SCALE", "USED", "baseline", "refusal_basis", "run_length", "trend", "unreadable_text"] diff --git a/backend/app/engine/nodes/analysis.py b/backend/app/engine/nodes/analysis.py index 382104a7..80501458 100644 --- a/backend/app/engine/nodes/analysis.py +++ b/backend/app/engine/nodes/analysis.py @@ -13,7 +13,10 @@ driver……),下游条件分支能按它走(周报「超出常规波动 (contract.ledger_entry,带 item),报告的写作目录编成 A{n};每项另发一条 analysis.end(运行详情、运行对比按项对齐)。 **一项算不了不让卡失败。** 算不了(历史不够、本期不完整、截断……)就是这一项的结论(status refused),照实记、发一条 -analysis_refused 提醒,接着做下一项。调用写错(列不存在、比率当可加……)是模板的毛病:整张卡失败,说清是哪一项、怎么改。 +analysis_refused 提醒,接着做下一项。模板定死了调用,这期的数据里没有要用的那一项(上一层没做成、下钻的段这期没有、期次 +这期没有、总数那一格越界或为空、要比的组这期没有,analysis.DATA_ERRORS)不是写错了,同样记成算不了的一项 +(run 的 refuse_data_errors)。结构性的调用错误(列不存在、遮罩列、比率当可加、重复行、参数写错……)是模板的毛病:整张卡 +失败,说清是哪一项、怎么改。 分析的结果不作口径卡的输入(D16):口径卡读 nodes.<卡>.results 的,按追不到原始数据处理(受管级发布时另有提示)。 """ @@ -74,8 +77,9 @@ async def run_analysis(state: GraphState, ctx: NodeContext) -> dict[str, Any]: who = f"分析项「{item_id}」({ANALYSIS_OP_LABEL[op]})" request = _request(item, state, ctx, who) try: - # 下钻(parent)只认这张卡里排在前面、已经做完的项:done 里此刻只有它们 - result = await analysis_ops.run(request, inputs.get, settings=settings, context=context, parents=done.get) + # 下钻(parent)只认这张卡里排在前面、已经做完的项:done 里此刻只有它们(算不了的也在,下钻那一项照实记成算不了) + result = await analysis_ops.run(request, inputs.get, settings=settings, context=context, parents=done.get, + refuse_data_errors=True) except CallError as e: raise NodeError(ctx.node.id, f"{who}:{e.message}。请在分析卡中改正这一项") from None except Exception as e: # noqa: BLE001 - 算子本身出错:说清是哪一项,原因照实 diff --git a/backend/tests/analysis_samples.py b/backend/tests/analysis_samples.py index 01fae6b0..e6a1b11d 100644 --- a/backend/tests/analysis_samples.py +++ b/backend/tests/analysis_samples.py @@ -52,8 +52,10 @@ CATALOGS = {"shop": {"orders": ORDERS}} async def analyze(op: str, mode: str | None = None, *, snaps: Sequence[Mapping[str, Any]], today: date = TODAY, catalogs: Mapping[str, Any] | None = None, probe: Any = None, facts: Any = None, parents: Mapping[str, Mapping[str, Any]] | None = None, settings: Mapping[str, Any] | None = None, - hypothesis: str | None = None, check: bool = True, **params: Any) -> dict[str, Any]: - """跑一次 analysis.run;check 时再复算一遍,核对和封存的逐字段相同(hypothesis 除外)。""" + hypothesis: str | None = None, check: bool = True, refuse_data_errors: bool = False, + **params: Any) -> dict[str, Any]: + """跑一次 analysis.run;check 时再复算一遍,核对和封存的逐字段相同(hypothesis 除外)。refuse_data_errors 同 run(分析卡 + 传真);复算一律按真。""" by_ref = {s["ref"]: s for s in snaps} request: dict[str, Any] = {"op": op, "mode": mode, "params": params} if hypothesis is not None: @@ -64,7 +66,7 @@ async def analyze(op: str, mode: str | None = None, *, snaps: Sequence[Mapping[s if facts is not None: context["facts"] = facts result = await analysis.run(request, by_ref.get, settings=effective_settings(settings), context=context, - parents=(parents or {}).get) + parents=(parents or {}).get, refuse_data_errors=refuse_data_errors) if check: again = recomputed(result, snaps, parents) assert same(result, again), _diff(result, again) diff --git a/backend/tests/fixtures/analysis_ops.py b/backend/tests/fixtures/analysis_ops.py index c9371b10..a785ed4c 100644 --- a/backend/tests/fixtures/analysis_ops.py +++ b/backend/tests/fixtures/analysis_ops.py @@ -30,7 +30,7 @@ class FakeOps: monkeypatch.setattr(analysis_ops, "run", self.run) async def run(self, request: dict[str, Any], resolve: Any, *, settings: Any, context: Any, - parents: Any = None) -> dict[str, Any]: + parents: Any = None, refuse_data_errors: bool = False) -> dict[str, Any]: params = request.get("params") or {} names = [params[k] for k in ("query", "base_query", "target_query") if params.get(k)] snapshots = {name: resolve(name) for name in names} @@ -38,7 +38,7 @@ class FakeOps: if isinstance(params.get("parent"), dict) and parents is not None: parent = parents(params["parent"]["of"]) self.calls.append({"request": request, "snapshots": snapshots, "settings": dict(settings), "context": context, - "parent": parent, "parents": parents, + "parent": parent, "parents": parents, "refuse_data_errors": refuse_data_errors, "visible": {n: parents is not None and parents(n) is not None for n in self.peek}}) error = self.errors.pop(0) if self.errors else None if error is not None: diff --git a/backend/tests/test_analysis_card_data.py b/backend/tests/test_analysis_card_data.py new file mode 100644 index 00000000..755df707 --- /dev/null +++ b/backend/tests/test_analysis_card_data.py @@ -0,0 +1,70 @@ +"""分析卡里随数据变化的调用错误(复核 R6):模板定死了调用,这期的数据里没有要用的那一项(上一层没做成、下钻的段这期没有……) +记成算不了的一项,不让卡失败;结构性的调用错误照旧让卡失败并说清哪一项。接真算子跑(tests/test_analysis_e2e 的虚构零售库 +shop),每份结论按封存的工件复算、逐字段相同。""" +from __future__ import annotations + +from app.core import artifact_store +from app.engine.runner import run_manager +from tests.test_analysis_e2e import ( # noqa: F401 - engine_up、shop 是自动用的夹具 + BY_REGION, EAST, TOOL, TWO_WEEKS, WEEK, chain, engine_up, events, node, recomputes, shop, wait, +) + +ONLY_EAST = (f"SELECT region, {WEEK} AS week, SUM(amount) AS gmv FROM orders " + "WHERE region = '华东' AND day >= '2026-05-11' GROUP BY region, week ORDER BY region, week") +WHERE = {"id": "where", "op": "decompose", "mode": "additive", "from": "by_region", + "params": {"by": ["region"], **TWO_WEEKS}} +DRILL = {"id": "east", "op": "decompose", "mode": "additive", "from": "east_q", + "params": {"by": ["channel"], **TWO_WEEKS, "parent": {"of": "where", "key": ["华东"]}}} + + +def card_graph(region_sql, items): + return chain(node("start", "input"), node("by_region", "tool", tool=TOOL, args={"sql": region_sql}), + node("east_q", "tool", tool=TOOL, args={"sql": EAST}), + node("card", "analysis", items=items), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.card.results.east.phrase }}"}])) + + +async def _card(region_sql, items): + run = await run_manager.start(graph=card_graph(region_sql, items), input_payload={}) + row = await wait(run.id) + return row + + +async def test_a_drill_under_a_refused_parent_is_refused_and_the_card_goes_on(): + row = await _card(ONLY_EAST, [WHERE, DRILL]) + assert row.status == "succeeded", row.error + [finished] = await events(row.id, "node.finished", "card") + results = artifact_store.load(finished["artifact"])["results"] + assert results["where"]["status"] == "refused" and results["east"]["status"] == "refused" + east = artifact_store.load(results["east"]["artifact"]) + assert east["reason_code"] == "parent_refused" + assert east["reason"] == "上一层「where」没有做成(按 region 只有 1 段,至少要 2 段才拆得开),不能在它下面下钻" + assert row.output["r"] == "变化拆解没有做成:" + east["reason"] + ends = {e["item"]: e for e in await events(row.id, "analysis.end", "card")} + assert ends["east"]["reason_code"] == "parent_refused" and ends["east"]["status"] == "refused" + warned = [e for e in await events(row.id, "log", "card") if e.get("code") == "analysis_refused"] + assert [e["item"] for e in warned] == ["where", "east"] + assert [(e["item"], e["status"]) for e in finished["evidence"]] == [("where", "refused"), ("east", "refused")] + recomputes(artifact_store.load(results["where"]["artifact"])) + recomputes(east) + + +async def test_a_segment_missing_this_time_is_refused_and_the_card_goes_on(): + drill = {**DRILL, "params": {**DRILL["params"], "parent": {"of": "where", "key": ["东北"]}}} + row = await _card(BY_REGION, [WHERE, drill]) + assert row.status == "succeeded", row.error + [finished] = await events(row.id, "node.finished", "card") + results = artifact_store.load(finished["artifact"])["results"] + assert results["where"]["status"] == "ok" and results["east"]["status"] == "refused" + east = artifact_store.load(results["east"]["artifact"]) + assert east["reason_code"] == "input_unavailable" and east["reason"] == "上一层「where」里没有「东北」这一段" + assert [i["role"] for i in east["inputs"]] == ["parent", "data"] + recomputes(east) + + +async def test_a_structural_call_error_still_fails_the_card_and_names_the_item(): + wrong = {**WHERE, "params": {**WHERE["params"], "by": ["channel"]}} + row = await _card(BY_REGION, [wrong]) + assert row.status == "failed" + assert "分析项「where」(变化拆解)" in row.error and "没有「channel」这一列" in row.error + assert "请在分析卡中改正这一项" in row.error diff --git a/backend/tests/test_analysis_run.py b/backend/tests/test_analysis_run.py index 836dba63..36c09a44 100644 --- a/backend/tests/test_analysis_run.py +++ b/backend/tests/test_analysis_run.py @@ -153,3 +153,97 @@ async def test_ledger_entries_name_the_real_input_snapshots(): def test_the_call_limit_error_is_spoken_with_the_limit(): error = analysis.exhausted({"max_calls": 5}) assert error.code == "max_calls" and error.message == "这个 Agent 这次执行最多做 5 次分析,已经用完:用已有的分析结论写,不再新做分析" + + +# -------------------------------------------------------------------------- +# 随数据变化的调用错误(分析卡传 refuse_data_errors):记成算不了的结论,复算也成立 +# -------------------------------------------------------------------------- + +ONE_REGION = snap("Q6", WEEKLY_SQL, ["region", "week", "gmv"], [["华东", "2026-W41", 820], ["华东", "2026-W42", 700]]) +TOTALS = snap("Q7", "SELECT week, SUM(amount) AS gmv FROM orders GROUP BY week", ["week", "gmv"], + [["2026-W42", 1926], ["2026-W41", None]]) +FACTORS = snap("Q8", "SELECT week, SUM(amount) AS gmv, COUNT(*) AS orders, AVG(amount) AS aov FROM orders " + "GROUP BY week", + ["week", "gmv", "orders", "aov"], [["2026-W41", 2120, 20, 106], ["2026-W42", None, 0, None]]) +ONE_GROUP = snap("Q9", "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region", + ["region", "refunds", "orders"], [["西南", 80, 1000]]) +WEEK_RATES = snap("Q10", "SELECT week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY week", + ["week", "refunds", "orders"], [["2026-W41", 30, 1000], ["2026-W42", None, 1000]]) +REFUSALS = [WEEKLY, SERIES, RATES, ONE_REGION, TOTALS, FACTORS, ONE_GROUP, WEEK_RATES] +BY_REGION = {"query": "Q2", "by": ["region"], "period": "week", "base": "2026-W41", "target": "2026-W42", + "measure": "gmv"} + + +async def _refused(op, mode=None, *, parents=None, **params): + """refuse_data_errors 为真跑两次:逐字节相同、形状照契约、复算逐字段相同(analyze 里按真复算);不传时照旧是调用错误。""" + one = await analyze(op, mode, snaps=REFUSALS, parents=parents, refuse_data_errors=True, **params) + two = await analyze(op, mode, snaps=REFUSALS, parents=parents, refuse_data_errors=True, **params) + assert json.dumps(one, sort_keys=True, ensure_ascii=False) == json.dumps(two, sort_keys=True, ensure_ascii=False) + _assert_shape(one) + assert one["status"] == "refused" and one["phrase"] == f"{c.OP_LABELS[op]}没有做成:{one['reason']}" + error = await call_error(op, mode, snaps=REFUSALS, parents=parents, **params) + assert error.code in analysis.DATA_ERRORS and error.reason == one["reason"] + return one + + +def test_the_data_errors_are_exported(): + assert analysis.DATA_ERRORS == {"parent_refused", "parent_key_missing", "period_not_found", "row_out_of_range", + "headline_empty", "unknown_group", "too_few_groups", "empty_cell"} + assert {"parent_refused", "input_unavailable"} <= set(c.REASONS) + + +async def test_a_drill_under_a_refused_parent_is_itself_refused(): + parent = await analyze("decompose", "additive", snaps=REFUSALS, **{**BY_REGION, "query": "Q6"}) + assert parent["reason_code"] == "too_few_segments" + result = await _refused("decompose", "additive", parents={"where": parent}, + **{**BY_REGION, "by": ["region"], "parent": {"of": "where", "key": ["华东"]}}) + assert result["reason_code"] == "parent_refused" + assert result["reason"] == "上一层「where」没有做成(按地区只有 1 段,至少要 2 段才拆得开),不能在它下面下钻" + # 上一层、取过的快照都记进输入(复算按它们走到同一处);参数照调用原样,阈值同这个模式自己算不了时 + assert [i["role"] for i in result["inputs"]] == ["parent", "data"] + assert result["inputs"][1] == {"ref": "Q2", "artifact": WEEKLY["artifact"], "role": "data", + "columns": ["region", "gmv", "week"], "rows": "all"} + assert result["params"]["parent"] == {"of": "where", "key": ["华东"]} + assert result["settings"] == {**{k: c.DEFAULTS[k] for k in ("main_share", "concentration", "offset_ratio", + "flat_pct", "table_top_k")}, "max_drill_depth": 3} + assert result["table"] is None and result["verdict"] is None + + +async def test_segments_periods_cells_and_groups_missing_this_time(): + parent = await analyze("decompose", "additive", snaps=REFUSALS, **BY_REGION) + key = await _refused("decompose", "additive", parents={"A1": parent}, + **{**BY_REGION, "parent": {"of": "A1", "key": ["东北"]}}) + assert key["reason_code"] == "input_unavailable" and key["reason"] == "上一层「A1」里没有「东北」这一段" + period = await _refused("decompose", "additive", **{**BY_REGION, "target": "2026-W43", "label": "销售额,主要来自华东"}) + assert period["reason"] == "期次列「week」里没有本期「2026-W43」" and period["params"]["label"] == "销售额" + row = await _refused("decompose", "additive", **BY_REGION, headline={"base": "Q7.r5.gmv", "target": "Q7.r0.gmv"}) + assert row["reason"] == "基期总数那一格(Q7.r5.gmv)不在:Q7 这次只有 2 行" + assert [i["role"] for i in row["inputs"]] == ["data", "headline_base"] + empty = await _refused("decompose", "additive", **BY_REGION, headline={"base": "Q7.r1.gmv", "target": "Q7.r0.gmv"}) + assert empty["reason"] == "基期总数那一格(Q7.r1.gmv)是空的" + factor = await _refused("decompose", "factor", query="Q8", period="week", base="2026-W41", target="2026-W42", + product="gmv", factors=["orders", "aov"]) + assert factor["reason"] == "本期的「gmv」那一格是空的(Q8 第 1 行)" + target = await _refused("baseline", query="Q1", period="week", measure="gmv", target="2020-01-06") + assert target["reason"] == "期次列「week」里没有本期「2020-01-06」(或那一期为空)" + assert target["settings"] == {k: c.DEFAULTS[k] for k in ("k_warn", "k_alert", "min_history", "max_history")} + side = await _refused("comparable", query="Q2", period="week", base="2026-W40", target="2026-W42") + assert side["reason"] == "期次列「week」里没有基期「2026-W40」" and side["settings"] == {} + rates = {"by": ["region"], "numerator": "refunds", "denominator": "orders"} + group = await _refused("differ", "rate", query="Q3", **rates, pairs=[["华东", "东北"]]) + assert group["reason"] == "分组列「region」里没有「东北」这一组" + assert group["settings"] == {"differ_alpha": 0.05, "min_expected": 5} + few = await _refused("differ", "rate", query="Q9", **rates) + assert few["reason"] == "分组列「region」里只有 1 组,至少要两组才比得了" + weeks = {"query": "Q10", "period": "week", "numerator": "refunds", "denominator": "orders"} + gone = await _refused("differ", "rate", **weeks, base="2026-W40", target="2026-W41") + assert gone["reason"] == "期次列「week」里没有基期「2026-W40」" + null = await _refused("differ", "rate", **weeks, base="2026-W41", target="2026-W42") + assert null["reason"] == "「refunds」那一格是空的(Q10 第 1 行)" + + +async def test_structural_call_errors_still_fail_even_when_data_errors_are_refused(): + for params in ({**BY_REGION, "by": ["channel"]}, {**BY_REGION, "by": ["region", "week"]}): + with pytest.raises(CallError) as e: + await analyze("decompose", "additive", snaps=REFUSALS, refuse_data_errors=True, check=False, **params) + assert e.value.code not in analysis.DATA_ERRORS and e.value.reason is None From 460af221f9c5d2a2bcf5a913ba2e42a785f582da Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 15:51:30 +0800 Subject: [PATCH 04/13] =?UTF-8?q?=E5=90=8C=E5=8F=A3=E5=BE=84=E5=85=88?= =?UTF-8?q?=E5=8E=BB=E6=8E=89=E6=9C=80=E5=A4=96=E5=B1=82=E6=8E=92=E5=BA=8F?= =?UTF-8?q?=E3=80=81=E6=8C=89=E5=8E=BB=E6=8E=89=E6=97=A5=E6=9C=9F=E7=9A=84?= =?UTF-8?q?=E5=86=99=E6=B3=95=E6=AF=94=EF=BC=88=E5=A4=8D=E6=A0=B8=20R7?= =?UTF-8?q?=EF=BC=89=EF=BC=9Asame=5Fquery=20=E4=B8=8D=E5=86=8D=E5=8F=AA?= =?UTF-8?q?=E5=9C=A8=E8=A7=84=E8=8C=83=E5=BD=A2=E5=BC=8F=E5=AE=8C=E5=85=A8?= =?UTF-8?q?=E7=9B=B8=E5=90=8C=E6=97=B6=E6=89=8D=E6=8A=8A=E6=97=A5=E6=9C=9F?= =?UTF-8?q?=E6=8D=A2=E6=88=90=E5=8D=A0=E4=BD=8D=E2=80=94=E2=80=94=E4=B8=A4?= =?UTF-8?q?=E8=BE=B9=E9=83=BD=E5=8E=BB=E6=8E=89=E6=9C=80=E5=A4=96=E5=B1=82?= =?UTF-8?q?=E7=9A=84=20ORDER=20BY=EF=BC=88=E5=90=8C=E4=B8=80=E5=B1=82?= =?UTF-8?q?=E6=9C=89=20LIMIT=20/=20OFFSET=20=E6=97=B6=E4=B8=8D=E5=8E=BB?= =?UTF-8?q?=EF=BC=9A=E6=8E=92=E5=BA=8F=E5=86=B3=E5=AE=9A=E5=8F=96=E5=93=AA?= =?UTF-8?q?=E5=87=A0=E8=A1=8C=EF=BC=89=EF=BC=8C=E6=97=B6=E9=97=B4=E6=9D=A1?= =?UTF-8?q?=E4=BB=B6=E7=9A=84=E7=95=8C=EF=BC=88=E5=AD=97=E9=9D=A2=E9=87=8F?= =?UTF-8?q?=E3=80=81=E6=8C=89=E8=BF=90=E8=A1=8C=E5=BD=93=E5=A4=A9=E7=8E=B0?= =?UTF-8?q?=E7=AE=97=E7=9A=84=E9=83=BD=E7=AE=97=EF=BC=89=E5=92=8C=E5=83=8F?= =?UTF-8?q?=E6=97=A5=E6=9C=9F=E7=9A=84=E5=AD=97=E9=9D=A2=E9=87=8F=E6=8D=A2?= =?UTF-8?q?=E6=88=90=E5=90=8C=E4=B8=80=E4=B8=AA=E5=8D=A0=E4=BD=8D=E5=90=8E?= =?UTF-8?q?=E8=A7=84=E8=8C=83=E5=8C=96=E5=86=8D=E6=AF=94=EF=BC=9A=E4=B8=80?= =?UTF-8?q?=E6=A0=B7=E5=B0=B1=E9=80=9A=E8=BF=87=EF=BC=8C=E5=8F=AA=E5=B7=AE?= =?UTF-8?q?=E5=AD=97=E9=9D=A2=E9=87=8F=E5=88=A4=E3=80=8C=E4=B8=A4=E8=BE=B9?= =?UTF-8?q?=E9=99=A4=E4=BA=86=E6=97=A5=E6=9C=9F=EF=BC=8C=E7=AD=9B=E9=80=89?= =?UTF-8?q?=E7=9A=84=E8=BF=98=E4=B8=8D=E4=B8=80=E6=A0=B7=E3=80=8D=EF=BC=8C?= =?UTF-8?q?=E8=BF=98=E4=B8=8D=E4=B8=80=E6=A0=B7=E6=89=8D=E4=BA=A4=E7=BB=99?= =?UTF-8?q?=20explain=5Fsql=5Fdiff=EF=BC=9B=E4=BA=A4=E7=BB=99=E5=AE=83?= =?UTF-8?q?=E7=9A=84=E4=B9=9F=E5=8E=BB=E6=8E=89=E6=9C=80=E5=A4=96=E5=B1=82?= =?UTF-8?q?=E6=8E=92=E5=BA=8F=E3=80=81=E7=8E=B0=E7=AE=97=E7=9A=84=E7=95=8C?= =?UTF-8?q?=E6=8D=A2=E6=88=90=E6=8C=89=E5=85=88=E5=90=8E=E7=BC=96=E5=8F=B7?= =?UTF-8?q?=E7=9A=84=E6=97=A5=E6=9C=9F=E5=AD=97=E9=9D=A2=E9=87=8F=EF=BC=88?= =?UTF-8?q?=E5=AE=83=E5=8F=AA=E8=AE=A4=E5=AD=97=E9=9D=A2=E9=87=8F=E7=9A=84?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E8=8C=83=E5=9B=B4=EF=BC=89=EF=BC=8C=E5=8F=AA?= =?UTF-8?q?=E5=B7=AE=E6=97=B6=E9=97=B4=E8=8C=83=E5=9B=B4=E3=80=81=E6=8E=92?= =?UTF-8?q?=E5=BA=8F=E7=9A=84=E7=85=A7=E6=97=A7=E7=AE=97=E5=90=8C=E5=8F=A3?= =?UTF-8?q?=E5=BE=84=E3=80=82=E5=8F=AA=E6=9C=89=E4=B8=80=E8=BE=B9=E5=B8=A6?= =?UTF-8?q?=20ORDER=20BY=E3=80=81=E4=B8=8A=E5=91=A8=E5=AF=B9=E6=9C=AC?= =?UTF-8?q?=E5=91=A8=E5=88=B0=E4=BB=8A=E5=A4=A9=E8=BF=99=E7=B1=BB=E7=8E=B0?= =?UTF-8?q?=E7=AE=97=E7=95=8C=E5=86=99=E6=B3=95=E4=B8=8D=E5=90=8C=E7=9A=84?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E8=90=BD=E8=BF=9B=E5=85=9C=E5=BA=95=E5=88=A4?= =?UTF-8?q?=E6=88=90=E7=BB=93=E6=9E=84=E4=B8=8D=E5=90=8C=EF=BC=8C=E4=B8=A4?= =?UTF-8?q?=E4=BB=BD=E5=BF=AB=E7=85=A7=E7=9A=84=E5=8F=98=E5=8C=96=E6=8B=86?= =?UTF-8?q?=E8=A7=A3=E4=B9=9F=E4=B8=8D=E5=86=8D=E5=9B=A0=E6=AD=A4=E4=B8=8D?= =?UTF-8?q?=E6=8B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/comparable.py | 67 +++++++++++++++++------ backend/tests/test_analysis_comparable.py | 52 ++++++++++++++++++ 2 files changed, 103 insertions(+), 16 deletions(-) diff --git a/backend/app/analysis/comparable.py b/backend/app/analysis/comparable.py index 76041bb6..bc157432 100644 --- a/backend/app/analysis/comparable.py +++ b/backend/app/analysis/comparable.py @@ -3,11 +3,12 @@ 五项检查,每项 pass / warn / fail / unknown(unknown 只写「核对不了」,不判不可比): -- 同口径:两份 SQL 一字不差 → pass;规范化(data/sql_semantics.normalize_sql)后只差字面量:把时间条件的界(data/coverage - 认出的时间谓词的界,整个表达式)和像日期的字面量换成同一个占位再比,一样就是只差日期 → pass,不一样就是比的两段不是 - 两期 → fail「两边除了日期,筛选的还不一样」;规范形式不同交给口径账本的 explain_sql_diff,等价 → pass,只差时间范围、排序 - → pass,否则 fail,写出差在哪些方面(只写方面,不写 SQL 原文里的值)。两份来自不同的数据源 fail。同一份快照里按期次分组 - 的天然同口径。 +- 同口径:两份 SQL 一字不差 → pass;否则两边都去掉最外层的 ORDER BY(没有 LIMIT 时排序不改结果),把时间条件的界(data/ + coverage 认出的时间谓词的界,整个表达式:字面量、按运行当天现算的都算)和像日期的字面量换成同一个占位,再规范化 + (data/sql_semantics.normalize_sql)比:一样就是只差日期 → pass;只差字面量就是比的两段不是两期 → fail「两边除了日期, + 筛选的还不一样」;还不一样交给口径账本的 explain_sql_diff(同样去掉最外层的 ORDER BY,现算的界换成按先后编号的日期字面量: + 它只认字面量的时间范围),等价 → pass,只差时间范围、排序 → pass,否则 fail,写出差在哪些方面(只写方面,不写 SQL 原文里 + 的值)。两份来自不同的数据源 fail。同一份快照里按期次分组的天然同口径。 - 窗口长度:用 data/coverage 的 filters 取业务日期列(目录的 business_date;没写就取唯一那一列时间谓词)上的区间,界是字面 量的直接解析,按运行当天现算的在库里算(Book 的 bound,记进 probes);算两边天数。整月对整月、整季对整季算一样长。 不相等:度量是流量 fail,比率、存量、没标的 warn。按期次分组的:两期粒度相同,一样长。 @@ -94,14 +95,26 @@ class Outcome: # -------------------------------------------------------------------------- +def _unordered(sql: str, read: str | None) -> Any: + """解析成语法树,去掉最外层的 ORDER BY(同一层有 LIMIT / OFFSET 时排序决定取哪几行,不去)。解析不了为 None。""" + from app.data import coverage + + tree = coverage.parse_sql(sql, read) + if tree is not None and tree.args.get("order") is not None and not tree.args.get("limit") \ + and not tree.args.get("offset"): + tree.set("order", None) + return tree + + def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None: - """时间条件的界、像日期的字面量换成同一个占位之后的规范形式(文本 + 剩下的字面量)。解析不了为 None。""" + """去掉最外层的 ORDER BY、时间条件的界和像日期的字面量换成同一个占位之后的规范形式(文本 + 剩下的字面量)。解析不了为 + None。""" from sqlglot import exp from app.data import coverage from app.data.sql_semantics import normalize_sql - tree = coverage.parse_sql(sql, read) + tree = _unordered(sql, read) if tree is None: return None for flt in coverage.filters(tree): @@ -117,6 +130,28 @@ def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None: return normalized.text, normalized.literals +def _for_diff(sql: str, read: str | None) -> str: + """交给 explain_sql_diff 的写法:去掉最外层的 ORDER BY;按运行当天现算的界换成按先后编号的日期字面量(它只认字面量的 + 时间范围,现算的界写法不一样会落进兜底的「写法不同」)。解析不了的原样交。""" + from sqlglot import exp + + from app.data import coverage + + tree = _unordered(sql, read) + if tree is None: + return sql + n = 0 + for flt in coverage.filters(tree): + for _, node, _ in flt.conds: + if node.parent is not None and not coverage.literal_bound(node)[0]: + n += 1 + node.replace(exp.Literal.string(f"1900-01-{n:02d}")) + try: + return tree.sql(dialect=read) + except Exception: # noqa: BLE001 - 写不回去就交原文 + return sql + + def same_query(base: Snap, target: Snap) -> Check: from app.data.sql_semantics import explain_sql_diff, normalize_sql @@ -133,17 +168,17 @@ def same_query(base: Snap, target: Snap) -> Check: if na.parse_error or nb.parse_error: check.reasons = ["SQL 解析不了"] return check - if na.text == nb.text: - ua, ub = _undated(a, read), _undated(b, read) - if ua is None or ub is None: - return check - if ua == ub: - check.status = "pass" - else: - check.status, check.reasons = "fail", ["两边除了日期,筛选的还不一样"] + ua, ub = _undated(a, read), _undated(b, read) + if ua is None or ub is None: + return check + if ua == ub: + check.status = "pass" + return check + if ua[0] == ub[0]: + check.status, check.reasons = "fail", ["两边除了日期,筛选的还不一样"] return check try: - diff = explain_sql_diff(a, b, read) + diff = explain_sql_diff(_for_diff(a, read), _for_diff(b, read), read) except Exception: # noqa: BLE001 - 比不了就当核对不了 return check aspects = [d.aspect for d in diff.items] diff --git a/backend/tests/test_analysis_comparable.py b/backend/tests/test_analysis_comparable.py index caf8d433..6dcb8bc3 100644 --- a/backend/tests/test_analysis_comparable.py +++ b/backend/tests/test_analysis_comparable.py @@ -204,3 +204,55 @@ async def test_one_snapshot_of_sqlite_weeks_uses_the_real_days_of_each_week(): plain = snap("Q1", sql, ["week", "orders"], rows, types={"week": "text", "orders": "number"}) unknown = await analyze("comparable", snaps=[plain], query="Q1", period="week", base="2026-W41", target="2026-W42") assert unknown["phrase"] == "两期可比,窗口长度、整期、星期构成核对不了" + + +def _same(a, b): + from app.analysis.comparable import same_query + from app.analysis.inputs import Snap + + return same_query(Snap("Q1", {"sql": a, "source": "shop"}, "sqlite"), + Snap("Q2", {"sql": b, "source": "shop"}, "sqlite")) + + +def test_the_same_query_ignores_the_outer_order_and_how_run_day_bounds_are_written(): + by_region = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '{a}' AND '{b}' " + "GROUP BY region{tail}") + # 只有一边带 ORDER BY:排序不改结果 + check = _same(by_region.format(a="2026-10-05", b="2026-10-11", tail=""), + by_region.format(a="2026-10-12", b="2026-10-18", tail=" ORDER BY gmv DESC")) + assert check.status == "pass" and check.reasons == [] + # 按运行当天现算的界写法不一样(上周对本周到今天) + window = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= {a} AND order_date < {b}" + assert _same(window.format(a="date('now', 'weekday 1', '-14 days')", b="date('now', 'weekday 1', '-7 days')"), + window.format(a="date('now', 'weekday 1', '-7 days')", b="date('now')")).status == "pass" + assert _same(window.format(a="date('now', 'weekday 1', '-14 days')", b="date('now', 'weekday 1', '-7 days')"), + "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', 'weekday 1', '-7 days')" + ).status == "pass" + + +def test_what_still_makes_two_queries_different(): + base = "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' GROUP BY region" + # 带 LIMIT 时排序决定取哪几段 + top = _same(base + " ORDER BY gmv DESC LIMIT 2", base.replace("2026-10-05", "2026-10-12") + " LIMIT 2") + assert top.status == "fail" + # 除了日期还差别的筛选、按别的时间列、多一个条件 + other = _same(base.replace("WHERE", "WHERE region = '华东' AND"), + base.replace("WHERE", "WHERE region = '华南' AND").replace("2026-10-05", "2026-10-12")) + assert (other.status, other.reasons) == ("fail", ["两边除了日期,筛选的还不一样"]) + paid = _same(base, base.replace("order_date", "paid_at").replace("2026-10-05", "2026-10-12")) + assert paid.status == "fail" and paid.reasons == ["两份查询的口径结构不同(差在时间列)"] + extra = _same(base, base.replace("GROUP BY", "AND status = 'paid' GROUP BY").replace("2026-10-05", "2026-10-12")) + assert extra.status == "fail" and "筛选条件" in extra.reasons[0] + + +async def test_decomposing_two_snapshots_that_differ_only_in_order_is_not_refused(): + a = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '2026-10-05' AND '2026-10-11' " + "GROUP BY region") + b = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '2026-10-12' AND '2026-10-18' " + "GROUP BY region ORDER BY gmv DESC") + base = snap("Q1", a, ["region", "gmv"], [["华东", 820], ["华南", 700], ["华北", 600]], types={"region": "text"}) + target = snap("Q2", b, ["region", "gmv"], [["华东", 700], ["华南", 666], ["华北", 560]], types={"region": "text"}) + result = await analyze("decompose", "additive", snaps=[base, target], probe=fixed_prober(RANGE), base_query="Q1", + target_query="Q2", by=["region"], measure="gmv", label="销售额") + assert result["status"] == "ok" and result["verdict"]["code"] == "driver" + assert not [c for c in result["caveats"] if c["code"] == "comparability"] From 427a6df23c108e8324b3123ed03cc7bad2f0fef5 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 15:56:16 +0800 Subject: [PATCH 05/13] =?UTF-8?q?=E5=B7=AE=E5=BC=82=E6=A3=80=E9=AA=8C?= =?UTF-8?q?=E6=8B=BF=E4=B8=8D=E5=88=B0=20p=20=E5=80=BC=E7=85=A7=E5=AE=9E?= =?UTF-8?q?=E8=AF=B4=E3=80=81=E6=95=8F=E6=84=9F=E6=80=A7=E5=8F=AA=E5=9C=A8?= =?UTF-8?q?=E7=9C=9F=E5=AE=9E=E7=9A=84=E5=90=84=E7=BB=84=E9=87=8C=E6=89=BE?= =?UTF-8?q?=E7=A6=BB=E7=BE=A4=E5=80=BC=EF=BC=88=E5=A4=8D=E6=A0=B8=20R8?= =?UTF-8?q?=E3=80=81R9=EF=BC=89=EF=BC=9A=E4=B8=80=E5=AF=B9=E9=83=BD?= =?UTF-8?q?=E6=8B=BF=E4=B8=8D=E5=88=B0=20p=20=E5=80=BC=EF=BC=88=E6=AF=94?= =?UTF-8?q?=E7=8E=87=E6=9C=89=E4=B8=80=E7=BB=84=E6=A0=B7=E6=9C=AC=E9=87=8F?= =?UTF-8?q?=E4=B8=BA=200=EF=BC=9B=E5=9D=87=E5=80=BC=E3=80=81=E9=80=90?= =?UTF-8?q?=E8=A1=8C=E6=9C=89=E4=B8=80=E7=BB=84=E4=B8=8D=E5=88=B0=202=20?= =?UTF-8?q?=E4=B8=AA=E6=A0=B7=E6=9C=AC=EF=BC=8C=E6=88=96=E5=9D=87=E5=80=BC?= =?UTF-8?q?=E3=80=81=E6=A0=87=E5=87=86=E5=B7=AE=E4=B8=BA=E7=A9=BA=EF=BC=89?= =?UTF-8?q?=E6=97=B6=E7=AE=97=E4=B8=8D=E4=BA=86=EF=BC=8C=E5=A5=91=E7=BA=A6?= =?UTF-8?q?=20REASONS=20=E5=8A=A0=20untestable=EF=BC=8C=E5=8E=9F=E5=9B=A0?= =?UTF-8?q?=E5=86=99=E5=93=AA=E4=B8=80=E7=BB=84=E6=80=8E=E4=B9=88=E4=BA=86?= =?UTF-8?q?=EF=BC=88=E3=80=8C=E5=8D=8E=E4=B8=9C=E5=8F=AA=E6=9C=89=201=20?= =?UTF-8?q?=E4=B8=AA=E6=A0=B7=E6=9C=AC=E3=80=81=E5=8D=8E=E5=8D=97=E5=8F=AA?= =?UTF-8?q?=E6=9C=89=201=20=E4=B8=AA=E6=A0=B7=E6=9C=AC=EF=BC=8C=E7=AE=97?= =?UTF-8?q?=E4=B8=8D=E4=BA=86=E5=B7=AE=E5=BC=82=E3=80=8D=EF=BC=89=EF=BC=8C?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E5=88=A4=20ok=20+=20=E5=8F=AF=E8=83=BD?= =?UTF-8?q?=E5=8F=AA=E6=98=AF=E6=A0=B7=E6=9C=AC=E6=B3=A2=E5=8A=A8=EF=BC=9B?= =?UTF-8?q?=E5=8F=AA=E6=9C=89=E4=B8=80=E9=83=A8=E5=88=86=E6=8B=BF=E4=B8=8D?= =?UTF-8?q?=E5=88=B0=E7=9A=84=EF=BC=8C=E9=82=A3=E5=87=A0=E5=AF=B9=E4=B8=8D?= =?UTF-8?q?=E7=AE=97=E8=BF=9B=E6=AF=94=E4=BA=86=E5=87=A0=E5=AF=B9=E3=80=81?= =?UTF-8?q?=E4=B8=8D=E5=8F=82=E4=B8=8E=E6=A0=A1=E6=AD=A3=EF=BC=8C=E8=A1=A8?= =?UTF-8?q?=E9=87=8C=E5=86=99=E6=B8=85=E5=93=AA=E4=B8=80=E7=BB=84=E6=80=8E?= =?UTF-8?q?=E4=B9=88=E4=BA=86=EF=BC=8C=E5=88=A4=E5=AE=9A=E8=AF=AD=E5=90=8E?= =?UTF-8?q?=E6=8E=A5=E3=80=8C=E5=8F=A6=E6=9C=89=20n=20=E5=AF=B9=E7=AE=97?= =?UTF-8?q?=E4=B8=8D=E4=BA=86=EF=BC=88=E2=80=A6=EF=BC=89=E3=80=8D=EF=BC=88?= =?UTF-8?q?=E5=A5=91=E7=BA=A6=20CAVEATS=20=E5=8A=A0=20untested=EF=BC=8C?= =?UTF-8?q?=E6=8E=92=E5=9C=A8=E5=88=A4=E5=AE=9A=E8=AF=AD=E9=99=84=E5=B8=A6?= =?UTF-8?q?=E8=AF=B4=E6=98=8E=E7=9A=84=E5=89=8D=E9=9D=A2=EF=BC=89=E3=80=82?= =?UTF-8?q?=E9=80=90=E8=A1=8C=E6=A8=A1=E5=BC=8F=E6=AF=94=E4=B8=89=E7=BB=84?= =?UTF-8?q?=E4=BB=A5=E4=B8=8A=E6=97=B6=EF=BC=8C=E7=A6=BB=E5=9D=87=E5=80=BC?= =?UTF-8?q?=E6=9C=80=E8=BF=9C=E7=9A=84=E9=82=A3=E4=B8=80=E7=AC=94=E5=8F=AA?= =?UTF-8?q?=E5=9C=A8=E7=9C=9F=E5=AE=9E=E7=9A=84=E5=90=84=E7=BB=84=E9=87=8C?= =?UTF-8?q?=E6=89=BE=EF=BC=88=E5=8E=9F=E6=9D=A5=E6=B1=A0=E5=AD=90=E9=87=8C?= =?UTF-8?q?=E6=B7=B7=E7=9D=80=E5=90=84=E5=AF=B9=E7=9A=84=E3=80=8C=E5=85=B6?= =?UTF-8?q?=E4=BD=99=E5=90=84=E7=BB=84=E5=90=88=E8=AE=A1=E3=80=8D=EF=BC=8C?= =?UTF-8?q?=E7=AC=AC=E4=B8=80=E7=BB=84=E4=BB=A5=E5=A4=96=E7=9A=84=E7=A6=BB?= =?UTF-8?q?=E7=BE=A4=E5=80=BC=E5=88=A0=E4=B8=8D=E5=88=B0=E5=AE=83=E8=87=AA?= =?UTF-8?q?=E5=B7=B1=E9=82=A3=E4=B8=80=E7=BB=84=EF=BC=8C=E6=BC=8F=E6=8A=A5?= =?UTF-8?q?=20outlier=5Fsensitive=EF=BC=89=EF=BC=8C=E5=8E=BB=E6=8E=89?= =?UTF-8?q?=E4=BB=A5=E5=90=8E=E7=94=B1=E5=90=84=E7=BB=84=E9=87=8D=E5=BB=BA?= =?UTF-8?q?=E6=AF=8F=E4=B8=80=E5=AF=B9=E7=9A=84=E3=80=8C=E5=85=B6=E4=BD=99?= =?UTF-8?q?=E3=80=8D=E5=90=88=E8=AE=A1=E5=86=8D=E6=A3=80=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/contract.py | 6 ++- backend/app/analysis/differ.py | 75 ++++++++++++++++++--------- backend/app/analysis/phrases.py | 11 +++- backend/tests/test_analysis_differ.py | 42 ++++++++++++++- 4 files changed, 104 insertions(+), 30 deletions(-) diff --git a/backend/app/analysis/contract.py b/backend/app/analysis/contract.py index 965e1bbb..9159875c 100644 --- a/backend/app/analysis/contract.py +++ b/backend/app/analysis/contract.py @@ -85,16 +85,18 @@ VERDICTS: dict[str, tuple[str, ...]] = { } #: 算不了的原因(细案 01 §8.2)。算不了也记成结论:写作者可以照实写「历史只有 5 周,判断不了是否异常」。最后两种是分析卡 #: 里随数据变化的调用错误记成的(analysis.run 的 refuse_data_errors,见 app/analysis 的 DATA_ERRORS):parent_refused 上一层 -#: 没做成;input_unavailable 这次的数据里没有要用的那一项(下钻的段、期次、总数那一格、要比的组……,reason 写是哪一项、哪个值) +#: 没做成;input_unavailable 这次的数据里没有要用的那一项(下钻的段、期次、总数那一格、要比的组……,reason 写是哪一项、哪个值)。 +#: untestable:差异检验一对都拿不到 p 值(两组各 1 个样本、样本量为 0……) REASONS: tuple[str, ...] = ( "truncated", "too_few_segments", "history_short", "partial_target", "target_gap", "series_gap", "period_unreadable", "zero_base", "non_positive", "incomparable_sides", "parent_refused", "input_unavailable", + "untestable", ) #: 附带说明的种类(caveats[].code) CAVEATS: tuple[str, ...] = ( "offset", "simpson", "small_denominator", "record_high", "record_low", "weekday_effect", "multiple_comparisons", "outlier_sensitive", "small_sample", "measure_unlabeled", "null_values", - "headline_basis", "comparability", "partial_periods", + "headline_basis", "comparability", "partial_periods", "untested", ) # -------------------------------------------------------------------------- diff --git a/backend/app/analysis/differ.py b/backend/app/analysis/differ.py index 12e6816b..668e31e5 100644 --- a/backend/app/analysis/differ.py +++ b/backend/app/analysis/differ.py @@ -16,6 +16,10 @@ 按 A − B 的正负);否则 no_evidence。用了 Fisher(比率)或有一组样本数小于 min_n(均值)→ small_sample。判定语不出现「p 值」 「显著性」:differs 说「差异站得住」,no_evidence 说「差异可能只是样本波动」,p 值、区间在表和证据面板里。 +拿不到 p 值的一对(比率有一组样本量为 0;均值、逐行有一组不到 2 个样本,或均值、标准差为空)不算进比了几对、不参与校正,表里 +照实写哪一组怎么了;一对都拿不到时算不了(untestable),不硬判「可能只是样本波动」;只有一部分拿不到的,判定语后接一句 +「另有 n 对算不了」(untested)。 + subjects 是判定说的那一对 [[A], [B]](只比一对时就是那一对,比了几对时是校正后最站得住的那一对;几对都不站得住时为空), direction 是 A 相对 B。 """ @@ -80,9 +84,13 @@ def _int(value: Fraction | None, snap: Snap, r: int, column: str) -> int: return int(value) +#: 「其余各组合计」的键(不是分组列里的值) +REST_KEY = ("__rest__",) + + def _pool(name: str, groups: Sequence[Group], mode: str) -> Group: """「其余各组合计」:比率把成功数、试验数加起来;均值按各组的均值、方差、样本数合并;逐行的值并在一起。""" - out = Group(("__rest__",), name, rows=[r for g in groups for r in g.rows]) + out = Group(REST_KEY, name, rows=[r for g in groups for r in g.rows]) if mode == "rate": out.x, out.n = sum(g.x for g in groups), sum(g.n for g in groups) return out @@ -137,6 +145,22 @@ def _test(pair: Pair, mode: str, min_expected: Fraction) -> None: pair.ci = stats.welch_interval(float(a.mean), float(a.var), a.n, float(b.mean), float(b.var), b.n) +def _gap(group: Group, mode: str) -> str | None: + """一组为什么算不了检验;算得了为 None。""" + if mode == "rate": + return ph.say(group.name, "的样本量为 0") if group.n == 0 else None + if group.n < 2: + return ph.say(group.name, "没有样本" if group.n == 0 else "只有 1 个样本") + if group.mean is None or group.var is None: + return ph.say(group.name, "的均值或标准差为空") + return None + + +def untested_text(pair: Pair, mode: str) -> str: + """这一对拿不到 p 值的原因:哪一组、怎么了(两组都有就都写)。""" + return "、".join(dict.fromkeys(why for g in (pair.a, pair.b) if (why := _gap(g, mode)))) or "样本不够" + + def _verdict(pairs: Sequence[Pair], alpha: Fraction) -> list[Pair]: """Holm 校正后站得住的那几对(按校正后的 p 从小到大,一样的按先后)。""" adjusted = stats.holm([p.p for p in pairs]) @@ -295,6 +319,12 @@ def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair] min_expected = env.threshold("min_expected") if mode == "rate" else Fraction(0) for pair in pairs: _test(pair, mode, min_expected) + tested = [p for p in pairs if p.p is not None] + untested = [p for p in pairs if p.p is None] + if not tested: + shown = list(dict.fromkeys(untested_text(p, mode) for p in pairs)) + return refuse(env, params, used, "untestable", + ",".join(shown[:3]) + ("等" if len(shown) > 3 else "") + ",算不了差异", criterion=criterion) hits = _verdict(pairs, alpha) caveats: list[dict[str, str]] = [] if mode == "rows": @@ -309,24 +339,26 @@ def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair] small = sorted((g for p in pairs for g in (p.a, p.b) if g.n < min_n), key=lambda g: g.n) if small: caveats.append(ph.caveat("small_sample", ph.small_sample_mean_text(small[0].name, small[0].n, min_n))) - tested = [p for p in pairs if p.p is not None] if len(tested) > 1: caveats.append(ph.caveat("multiple_comparisons", ph.multiple_text(len(tested)))) + if untested: + caveats.append(ph.caveat("untested", ph.untested_pairs_text(len(untested), + [untested_text(p, mode) for p in untested]))) fmt = (lambda v: ph.pct(v, signed=False)) if mode == "rate" else (lambda v: ph.amount(v)) if hits: code = "differs" head = hits[0] main = ph.differ_phrase([ph.differ_pair(p.a.name, p.b.name, label, fmt(p.a_value), fmt(p.b_value), p.diff > 0) for p in hits], # type: ignore[operator] - len(pairs)) + len(tested)) subjects = [[head.a.name], [head.b.name]] direction = sign_word(head.diff) # type: ignore[arg-type] else: code = "no_evidence" - head = pairs[0] if len(pairs) == 1 else None + head = tested[0] if len(tested) == 1 else None sizes = (ph.amount(head.a.n), ph.amount(head.b.n)) if head is not None else None main = ph.no_evidence_phrase(head.a.name if head else None, head.b.name if head else None, label, sizes, - len(pairs)) + len(tested)) subjects = [[head.a.name], [head.b.name]] if head is not None else [] direction = None value_name = "比率" if mode == "rate" else "均值" @@ -334,7 +366,7 @@ def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair] for p in pairs: flags = [] if p.p is None: - flags.append("样本不够,算不了") + flags.append(f"{untested_text(p, mode)},算不了") if p.test == "fisher": flags.append("样本少,用了精确检验") rows.append([p.a.name, p.b.name, canon(p.a_value), canon(p.b_value), canon(p.diff), @@ -342,7 +374,7 @@ def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair] TESTS.get(p.test or "") or None, canon(p.p), canon(p.p_adj), "、".join(flags) or None]) columns = ["a", "b", "a_value", "b_value", "diff", "ci_lo", "ci_hi", "a_n", "b_n", "test", "p", "p_adj", "flag"] # 组、对照组照分组列(期次列)在快照里的类型;和「其余各组合计」比时对照组那一列有一格是叫法,写 text - rest = any(p.b.key == ("__rest__",) for p in pairs) + rest = any(p.b.key == REST_KEY for p in pairs) table = {"columns": columns, "labels": {"a": "组", "b": "对照组", "a_value": f"组的{value_name}", "b_value": f"对照组的{value_name}", "diff": "差", "ci_lo": "差的区间下限", "ci_hi": "差的区间上限", "a_n": "组的样本量", @@ -350,37 +382,32 @@ def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair] "rows": rows, "column_types": column_types(columns, dims={"a": kind, "b": "text" if rest else kind}, texts=("test", "flag"))} - shown = head if head is not None else pairs[0] + shown = head if head is not None else tested[0] values = {"a": canon(shown.a_value), "b": canon(shown.b_value), "diff": canon(shown.diff), "ci_lo": canon(shown.ci[0]) if shown.ci else None, "ci_hi": canon(shown.ci[1]) if shown.ci else None, "p": canon(shown.p), "p_adj": canon(shown.p_adj), "n_a": str(shown.a.n), "n_b": str(shown.b.n), "test": shown.test, "pairs": str(len(pairs)), "differs": str(len(hits))} verdict = {"code": code, "subjects": subjects, "direction": direction} - phrase = ph.with_caveats(main, caveats, skip=("multiple_comparisons",) if len(pairs) > 1 else ()) + phrase = ph.with_caveats(main, caveats, skip=("multiple_comparisons",) if len(tested) > 1 else ()) return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, phrase=phrase, caveats=caveats) def _sensitivity(pairs: list[Pair], mode: str, alpha: Fraction, min_expected: Fraction, differs: bool) -> tuple[bool, bool] | None: - """rows 模式的敏感性:去掉比较的各组里离均值最远的那一个值重算,判定变了交回 (去掉的是不是偏大的那一笔, 现在站不站得住)。""" - everything = [(g, i, v) for g in dict.fromkeys(g for p in pairs for g in (p.a, p.b)) for i, v in enumerate(g.values)] + """rows 模式的敏感性:去掉比较的各组里离均值最远的那一个值重算,判定变了交回 (去掉的是不是偏大的那一笔, 现在站不站得住)。 + 只在真实的各组里找(「其余各组合计」是它们的值并在一起,不另算一份),去掉以后由各组重建每一对的「其余」合计。""" + groups = list(dict.fromkeys(g for p in pairs for g in (p.a, p.b) if g.key != REST_KEY)) + everything = [(g, i, v) for g in groups for i, v in enumerate(g.values)] if len(everything) < 3: return None mean = sum((v for _, _, v in everything), Fraction(0)) / len(everything) group, index, value = max(everything, key=lambda t: abs(t[2] - mean)) - trimmed = [] - for p in pairs: - def cut(g: Group) -> Group: - if g is not group and not (g.key == ("__rest__",) and any(v is value for v in g.values)): - return g - values = list(g.values) - for k, v in enumerate(values): - if v is value: - del values[k] - break - return _moments(Group(g.key, g.name, rows=g.rows, values=values)) - trimmed.append(Pair(cut(p.a), cut(p.b))) + cut = {g: g for g in groups} + cut[group] = _moments(Group(group.key, group.name, rows=group.rows, + values=[v for k, v in enumerate(group.values) if k != index])) + trimmed = [Pair(cut[p.a], _pool(p.b.name, [cut[g] for g in groups if g is not p.a], mode) + if p.b.key == REST_KEY else cut[p.b]) for p in pairs] for p in trimmed: _test(p, mode, min_expected) now = bool(_verdict(trimmed, alpha)) @@ -388,4 +415,4 @@ def _sensitivity(pairs: list[Pair], mode: str, alpha: Fraction, min_expected: Fr return (value > mean, now) if now != differs else None -__all__ = ["TESTS", "differ", "refusal_basis", "used_settings"] +__all__ = ["REST_KEY", "TESTS", "differ", "refusal_basis", "untested_text", "used_settings"] diff --git a/backend/app/analysis/phrases.py b/backend/app/analysis/phrases.py index c379364a..7b84de8a 100644 --- a/backend/app/analysis/phrases.py +++ b/backend/app/analysis/phrases.py @@ -32,8 +32,8 @@ from app.analysis.contract import OP_LABELS from app.analysis.numbers import fixed, to_fraction #: 附带说明进判定语的先后;不在这里的(measure_unlabeled、headline_basis、partial_periods)只在证据面板 -PHRASE_ORDER = ("comparability", "simpson", "offset", "outlier_sensitive", "small_sample", "record_high", "record_low", - "multiple_comparisons", "weekday_effect", "small_denominator", "null_values") +PHRASE_ORDER = ("comparability", "untested", "simpson", "offset", "outlier_sensitive", "small_sample", "record_high", + "record_low", "multiple_comparisons", "weekday_effect", "small_denominator", "null_values") #: 判定语里最多接几条附带说明 PHRASE_CAVEATS = 2 #: 消毒后的叫法最多几个字 @@ -343,6 +343,12 @@ def multiple_text(pairs: int) -> str: return f"比了 {pairs} 对,已按次数校正" +def untested_pairs_text(count: int, reasons: Sequence[str]) -> str: + """差异检验里拿不到 p 值的那几对(不算进比了几对):另有 1 对算不了(华北的样本量为 0)。""" + shown = list(dict.fromkeys(reasons)) + return f"另有 {count} 对算不了({','.join(shown[:3])}{'等' if len(shown) > 3 else ''})" + + def outlier_text(largest: bool, now_differs: bool) -> str: which = "最大" if largest else "最小" return f"去掉{which}的一笔后,差异{'变得站得住' if now_differs else '不再站得住'}" @@ -567,6 +573,7 @@ __all__ = [ "decompose_flat", "decompose_unclosed", "differ_pair", "differ_phrase", "dims_text", "factor_both", "factor_driver", "key_text", "multiple_text", "names", "no_evidence_phrase", "null_periods_text", "null_segments_text", "number", "offset_text", "outlier_text", "partial_periods_text", "pct", "pick_label", + "untested_pairs_text", "points", "ratio_both", "ratio_change", "ratio_mix", "ratio_rate", "ratio_value", "record_text", "refused", "say", "share_words", "simpson_text", "small_denominator_text", "small_sample_mean_text", "small_sample_rate_text", "threshold_pct", diff --git a/backend/tests/test_analysis_differ.py b/backend/tests/test_analysis_differ.py index f07d4e6c..bfb5d7d5 100644 --- a/backend/tests/test_analysis_differ.py +++ b/backend/tests/test_analysis_differ.py @@ -136,6 +136,44 @@ async def test_two_periods_of_the_same_group(): async def test_a_group_with_one_value_cannot_be_tested(): s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], [["线上", 5], ["线下", 4], ["线下", 6]]) result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount") - assert result["verdict"]["code"] == "no_evidence" and result["values"]["p"] is None - assert result["table"]["rows"][0][-1] == "样本不够,算不了" + assert result["status"] == "refused" and result["reason_code"] == "untestable" + assert result["reason"] == "线上只有 1 个样本,算不了差异" + assert result["phrase"] == "差异检验没有做成:线上只有 1 个样本,算不了差异" + + +async def test_no_pair_with_a_p_value_is_refused_not_said_to_be_noise(): + sql = "SELECT region, AVG(amount) AS m, 0 AS sd, COUNT(*) AS n FROM orders GROUP BY 1" + means = snap("Q1", sql, ["region", "m", "sd", "n"], [["华东", 120, 0, 1], ["华南", 80, 0, 1]]) + mean = await analyze("differ", "mean", snaps=[means], query="Q1", by="region", mean="m", sd="sd", n="n", + label="客单价") + assert mean["reason_code"] == "untestable" and mean["reason"] == "华东只有 1 个样本、华南只有 1 个样本,算不了差异" + assert mean["verdict"] is None and mean["settings"] == {"differ_alpha": 0.05, "min_n": 30} + rows = snap("Q2", "SELECT region, amount FROM orders", ["region", "amount"], [["华东", 120], ["华南", 80]]) + one = await analyze("differ", "rows", snaps=[rows], query="Q2", by="region", value="amount") + assert one["reason"] == "华东只有 1 个样本、华南只有 1 个样本,算不了差异" + zero = await rate([["华东", 0, 0], ["华南", 3, 10]]) + assert zero["reason_code"] == "untestable" and zero["reason"] == "华东的样本量为 0,算不了差异" + + +async def test_pairs_without_a_p_value_are_named_and_not_counted(): + result = await rate([["西南", 80, 1000], ["华东", 40, 1000], ["华北", 0, 0]]) + rows = {r[0]: r for r in result["table"]["rows"]} + assert rows["华北"][-1] == "华北的样本量为 0,算不了" and rows["华北"][10] is None + note = next(c for c in result["caveats"] if c["code"] == "untested") + assert note["text"] == "另有 1 对算不了(华北的样本量为 0)" + # 华北一单都没有:西南的其余合计就是华东,两对都站得住;华北那一对不算进「比了几对」 + assert result["phrase"] == ("比了 2 对,2 对差异站得住:西南的退款率(8.0%)高于其余各地区(4.0%);华东的退款率(4.0%)" + "低于其余各地区(8.0%);另有 1 对算不了(华北的样本量为 0)") + assert {"code": "multiple_comparisons", "text": "比了 2 对,已按次数校正"} in result["caveats"] + + +async def test_the_outlier_is_looked_for_in_the_real_groups_not_in_the_pooled_rest(): + # 三组各和其余合计比:最远的那一笔在第二组(华南)。只在真实的各组里找、由各组重建「其余」合计,去掉它以后华南高出来就站得住 + base = [10, 11, 9, 10, 12, 8, 10, 11, 10, 9, 11, 10] + rows = ([["华东", v] for v in base] + [["华南", v + 2] for v in base[:-1]] + [["华南", 100]] + + [["华北", v] for v in base]) + s = snap("Q1", "SELECT region, amount FROM orders", ["region", "amount"], rows) + result = await analyze("differ", "rows", snaps=[s], query="Q1", by="region", value="amount", label="客单价") + assert result["verdict"]["code"] == "no_evidence" + assert {"code": "outlier_sensitive", "text": "去掉最大的一笔后,差异变得站得住"} in result["caveats"] assert math.isfinite(float(result["values"]["diff"])) From fcbcfc9aefc86f929f5a60caf930ef0dc1cbf0f7 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 15:57:51 +0800 Subject: [PATCH 06/13] =?UTF-8?q?=E5=88=A4=E5=AE=9A=E8=AF=AD=E7=85=A7?= =?UTF-8?q?=E6=96=B9=E5=90=91=E6=8E=A5=E6=9C=AB=E5=B0=BE=E8=BF=9E=E7=BB=AD?= =?UTF-8?q?=E6=9C=9F=E6=95=B0=E3=80=81=E8=BE=9B=E6=99=AE=E6=A3=AE=E7=9A=84?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E5=8F=98=E5=8C=96=E5=8F=AA=E8=AF=B4=E4=B8=80?= =?UTF-8?q?=E9=81=8D=EF=BC=88=E5=A4=8D=E6=A0=B8=20R14=E3=80=81R21=EF=BC=89?= =?UTF-8?q?=EF=BC=9A=E8=B6=8B=E5=8A=BF=E6=9C=AB=E5=B0=BE=E8=BF=9E=E7=BB=AD?= =?UTF-8?q?=E5=90=8C=E5=90=91=E7=9A=84=E6=9C=9F=E6=95=B0=E5=92=8C=E8=B6=8B?= =?UTF-8?q?=E5=8A=BF=E5=90=8C=E5=90=91=E6=97=B6=E6=8E=A5=E3=80=8C=EF=BC=8C?= =?UTF-8?q?=E5=B7=B2=E8=BF=9E=E7=BB=AD=203=20=E5=91=A8=E4=B8=8B=E9=99=8D?= =?UTF-8?q?=E3=80=8D=EF=BC=8C=E5=8F=8D=E5=90=91=E6=97=B6=E6=8E=A5=E3=80=8C?= =?UTF-8?q?=EF=BC=8C=E4=BD=86=E6=9C=80=E8=BF=91=E5=B7=B2=E8=BF=9E=E7=BB=AD?= =?UTF-8?q?=203=20=E5=91=A8=E4=B8=8A=E5=8D=87=E3=80=8D=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E5=87=BA=E7=8E=B0=E3=80=8C=E5=91=88=E4=B8=8B=E9=99=8D?= =?UTF-8?q?=E8=B6=8B=E5=8A=BF=E2=80=A6=E2=80=A6=E5=B7=B2=E8=BF=9E=E7=BB=AD?= =?UTF-8?q?=203=20=E5=91=A8=E4=B8=8A=E5=8D=87=E3=80=8D=E3=80=82=E6=AF=94?= =?UTF-8?q?=E7=8E=87=E6=8B=86=E8=A7=A3=E5=88=A4=E5=9C=A8=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E5=8F=98=E5=8C=96=E4=B8=8A=EF=BC=88mix=5Fdriven=EF=BC=89?= =?UTF-8?q?=E6=97=B6=E6=AD=A3=E6=96=87=E5=B7=B2=E7=BB=8F=E8=AF=B4=E4=BA=86?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E5=8F=98=E5=8C=96=E3=80=81=E7=82=B9=E4=BA=86?= =?UTF-8?q?=E9=82=A3=E4=B8=80=E6=AE=B5=EF=BC=8C=E8=BE=9B=E6=99=AE=E6=A3=AE?= =?UTF-8?q?=E9=82=A3=E4=B8=80=E5=8F=A5=E5=8F=AA=E8=AF=B4=E3=80=8C=E5=90=84?= =?UTF-8?q?=E6=AE=B5=E7=9A=84=E9=80=80=E6=AC=BE=E7=8E=87=E5=A4=A7=E5=A4=9A?= =?UTF-8?q?=E5=9C=A8=E4=B8=8B=E9=99=8D=EF=BC=8C=E6=95=B4=E4=BD=93=E5=8D=B4?= =?UTF-8?q?=E4=B8=8A=E5=8D=87=E3=80=8D=EF=BC=9B=E5=88=AB=E7=9A=84=E5=88=A4?= =?UTF-8?q?=E5=AE=9A=E7=85=A7=E6=97=A7=E5=B8=A6=E4=B8=8A=E5=93=AA=E4=B8=80?= =?UTF-8?q?=E6=AE=B5=E5=8D=A0=E6=AF=94=E4=BB=8E=E5=A4=9A=E5=B0=91=E5=8F=98?= =?UTF-8?q?=E5=88=B0=E5=A4=9A=E5=B0=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/decompose.py | 4 +++- backend/app/analysis/phrases.py | 15 ++++++++++----- backend/tests/test_analysis_decompose.py | 7 +++++-- backend/tests/test_analysis_phrases.py | 15 +++++++++++++++ backend/tests/test_analysis_series.py | 6 ++++++ 5 files changed, 39 insertions(+), 8 deletions(-) diff --git a/backend/app/analysis/decompose.py b/backend/app/analysis/decompose.py index 1d42bb88..18520067 100644 --- a/backend/app/analysis/decompose.py +++ b/backend/app/analysis/decompose.py @@ -577,9 +577,11 @@ def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[ against = sum(((by_key[x.key][0] + by_key[x.key][1]) / 2 for x in eligible if (by_key[x.key][3] - by_key[x.key][2]) * sign < 0), Fraction(0)) if weight > 0 and against / weight >= SIMPSON_SHARE: + # 判在结构变化上(mix_driven)时判定语正文已经说了结构变化、点了这一段,附带说明只说各段和整体反着变 star = max(live, key=lambda x: sign * by_key[x.key][4]) caveats.append(ph.caveat("simpson", ph.simpson_text(label, delta > 0, ph.key_text(star.key), - by_key[star.key][0], by_key[star.key][1]))) + by_key[star.key][0], by_key[star.key][1], + shift=code != "mix_driven"))) if small: caveats.append(ph.caveat("small_denominator", ph.small_denominator_text( [ph.key_text(x.key) for x in small], den_label, int(env.settings["min_denominator"])))) diff --git a/backend/app/analysis/phrases.py b/backend/app/analysis/phrases.py index 7b84de8a..e359c473 100644 --- a/backend/app/analysis/phrases.py +++ b/backend/app/analysis/phrases.py @@ -301,13 +301,16 @@ def offset_text(delta_total: Any, gross_up: Any, gross_down: Any) -> str: return f"下降的段合计 {down},上升的段合计 {up},互相抵消了一部分" -def simpson_text(label: str, overall_up: bool, subject: str, w_base: Any, w_target: Any) -> str: - """辛普森:各段的比率大多和整体反着变,整体的变化来自结构。""" +def simpson_text(label: str, overall_up: bool, subject: str, w_base: Any, w_target: Any, *, shift: bool = True) -> str: + """辛普森:各段的比率大多和整体反着变,整体的变化来自结构。判定语正文已经说了结构变化、点了那一段(mix_driven)时 + shift 为假,只说各段和整体反着变,结构变化不说第二遍。""" seg = "上升" if not overall_up else "下降" whole = "上升" if overall_up else "下降" + head = say("各段的", label, f"大多在{seg},整体却{whole}") + if not shift: + return head move = "升" if _q(w_target) > _q(w_base) else "降" - return say(f"各段的", label, f"大多在{seg},整体却{whole}:变化来自结构,", subject, - f"占比从 {pct(w_base, signed=False)} {move}到 {pct(w_target, signed=False)}") + return say(head, ":变化来自结构,", subject, f"占比从 {pct(w_base, signed=False)} {move}到 {pct(w_target, signed=False)}") def small_denominator_text(segments: Sequence[str], denominator: str, minimum: int) -> str: @@ -478,13 +481,15 @@ def baseline_flat_phrase(verdict: str, label: str, current: str, n: int, unit: s def trend_phrase(verdict: str, label: str, n: int, unit: str, per: str, slope: str | None, run: tuple[int, str] | None) -> str: + """趋势。末尾连续同向的期数(run)和趋势同向时接「,已连续 3 周下降」,反向时接「,但最近已连续 3 周上升」。""" if verdict == "no_trend": return say(f"过去 {n} {unit}", label, "没有明显的上升或下降趋势") word = "上升" if verdict == "rising" else "下降" text = say(f"过去 {n} {unit}", label, f"呈{word}趋势") + (f"(每{per}约 {slope})" if slope else "") if run is not None: length, direction = run - text += f",已连续 {length} {unit}{'上升' if direction == 'up' else '下降'}" + same = (direction == "up") == (verdict == "rising") + text += f",{'' if same else '但最近'}已连续 {length} {unit}{'上升' if direction == 'up' else '下降'}" return text diff --git a/backend/tests/test_analysis_decompose.py b/backend/tests/test_analysis_decompose.py index 9710e192..f651c35d 100644 --- a/backend/tests/test_analysis_decompose.py +++ b/backend/tests/test_analysis_decompose.py @@ -301,9 +301,12 @@ async def test_simpson_each_channel_improves_but_the_mix_makes_the_whole_worse() rows = [["线上", "W41", 50, 500], ["线上", "W42", 90, 1000], ["线下", "W41", 10, 1000], ["线下", "W42", 4, 500]] result = await ratio(rows, label="退款率") assert result["verdict"] == {"code": "mix_driven", "subjects": [["线上"]], "direction": "up"} - assert result["phrase"].startswith("退款率上升 2.3 个百分点(4.0% 到 6.3%),主要来自结构变化:线上占比从 33.3% 到 66.7%") + # 结构变化只说一遍:正文说了结构变化、点了线上,辛普森那一句只说各段和整体反着变 + assert result["phrase"] == ("退款率上升 2.3 个百分点(4.0% 到 6.3%),主要来自结构变化:线上占比从 33.3% 到 66.7%;" + "各段的退款率大多在下降,整体却上升") simpson = next(c for c in result["caveats"] if c["code"] == "simpson") - assert simpson["text"] == "各段的退款率大多在下降,整体却上升:变化来自结构,线上占比从 33.3% 升到 66.7%" + assert simpson["text"] == "各段的退款率大多在下降,整体却上升" + assert result["phrase"].count("结构") == 1 values = result["values"] assert Fraction(values["mix_total"]) + Fraction(values["rate_total"]) == pytest.approx(Fraction(values["delta"])) assert result["table"]["columns"][:5] == ["channel", "n_base", "n_target", "d_base", "d_target"] diff --git a/backend/tests/test_analysis_phrases.py b/backend/tests/test_analysis_phrases.py index 108becf1..e87aa2f6 100644 --- a/backend/tests/test_analysis_phrases.py +++ b/backend/tests/test_analysis_phrases.py @@ -141,3 +141,18 @@ def test_criteria_are_written_with_the_actual_thresholds(): assert "3 倍以上算明显超出" in ph.criterion_baseline(DEFAULTS, 12, "周") assert "小于 0.05" in ph.criterion_trend(DEFAULTS, 10, "周") assert "Fisher" in ph.criterion_differ(DEFAULTS, "rate") and "Welch" in ph.criterion_differ(DEFAULTS, "mean") + + + +def test_the_run_at_the_end_of_a_trend_says_whether_it_goes_the_same_way(): + assert ph.trend_phrase("falling", "销售额", 10, "周", "周", "-2.0%", (3, "down")) == \ + "过去 10 周销售额呈下降趋势(每周约 -2.0%),已连续 3 周下降" + assert ph.trend_phrase("falling", "销售额", 10, "周", "周", "-2.0%", (3, "up")) == \ + "过去 10 周销售额呈下降趋势(每周约 -2.0%),但最近已连续 3 周上升" + assert ph.trend_phrase("rising", "订单数", 8, "个月", "月", None, (4, "down")) == \ + "过去 8 个月订单数呈上升趋势,但最近已连续 4 个月下降" + + +def test_simpson_says_the_shift_only_when_the_phrase_does_not(): + assert ph.simpson_text("退款率", True, "线上", 0.3, 0.5) == "各段的退款率大多在下降,整体却上升:变化来自结构,线上占比从 30.0% 升到 50.0%" + assert ph.simpson_text("退款率", True, "线上", 0.3, 0.5, shift=False) == "各段的退款率大多在下降,整体却上升" diff --git a/backend/tests/test_analysis_series.py b/backend/tests/test_analysis_series.py index 671598a3..f52ae0c6 100644 --- a/backend/tests/test_analysis_series.py +++ b/backend/tests/test_analysis_series.py @@ -313,3 +313,9 @@ async def test_sequence_periods_say_completeness_cannot_be_checked(): result = await baseline(rows, basis="level") assert {"code": "comparability", "text": "没能核对本期的数据是否到齐(期次不是日期,不知道是哪几天)"} \ in result["caveats"] + + +async def test_a_falling_trend_whose_last_weeks_rise_says_but(): + falling = await trend(weeks([200, 196, 190, 188, 181, 176, 170, 165, 160, 162, 165, 168])) + assert falling["verdict"]["code"] == "falling" and falling["values"]["run_direction"] == "up" + assert falling["phrase"].endswith(",但最近已连续 3 周上升") From efff2ac4f8f60706d9920b9e1dcef3801195cd6f Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 15:59:11 +0800 Subject: [PATCH 07/13] =?UTF-8?q?=E9=97=AD=E5=90=88=E5=AE=B9=E5=B7=AE?= =?UTF-8?q?=EF=BC=9A=E6=95=B4=E6=95=B0=E7=9A=84=E6=B5=AE=E7=82=B9=E6=8C=89?= =?UTF-8?q?=E4=B8=AA=E4=BD=8D=E3=80=81=E4=B8=8B=E9=92=BB=E5=AF=B9=E4=B8=8A?= =?UTF-8?q?=E4=B8=80=E5=B1=82=E7=9A=84=E7=B2=BE=E7=A1=AE=E5=80=BC=E6=8C=89?= =?UTF-8?q?=E7=B2=BE=E7=A1=AE=E6=AF=94=EF=BC=88=E5=A4=8D=E6=A0=B8=20R15?= =?UTF-8?q?=EF=BC=89=EF=BC=9A=E6=80=BB=E6=95=B0=E9=82=A3=E4=B8=80=E6=A0=BC?= =?UTF-8?q?=E6=98=AF=E6=95=B4=E6=95=B0=E7=9A=84=E6=B5=AE=E7=82=B9=EF=BC=88?= =?UTF-8?q?SQLite=20=E7=9A=84=20ROUND()=20=E4=BA=A4=E5=9B=9E=202120.0?= =?UTF-8?q?=EF=BC=89=E6=97=B6=E6=8C=89=E5=8F=96=E5=88=B0=E4=B8=AA=E4=BD=8D?= =?UTF-8?q?=E7=AE=97=E5=AE=B9=E5=B7=AE=EF=BC=88=E5=B7=AE=200.4=20=E4=B9=9F?= =?UTF-8?q?=E9=97=AD=E5=90=88=EF=BC=89=EF=BC=8C=E6=9C=80=E7=9F=AD=E5=86=99?= =?UTF-8?q?=E6=B3=95=E9=87=8C=E7=9A=84=E3=80=8C.0=E3=80=8D=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E5=BD=93=E4=B8=80=E4=BD=8D=E5=B0=8F=E6=95=B0=EF=BC=9B?= =?UTF-8?q?=E4=B8=8B=E9=92=BB=E6=97=B6=E5=AF=B9=E4=B8=8A=E4=B8=80=E5=B1=82?= =?UTF-8?q?=E9=82=A3=E4=B8=80=E6=AE=B5=E7=9A=84=E5=80=BC=E6=98=AF=E4=B8=8A?= =?UTF-8?q?=E4=B8=80=E5=B1=82=E7=AE=97=E5=87=BA=E6=9D=A5=E7=9A=84=E7=B2=BE?= =?UTF-8?q?=E7=A1=AE=E5=80=BC=E3=80=81=E6=B2=A1=E6=9C=89=E5=8F=96=E6=95=B4?= =?UTF-8?q?=E8=BF=87=EF=BC=8C=E5=8F=AA=E5=AE=B9=E6=B5=AE=E7=82=B9=E7=AE=97?= =?UTF-8?q?=E5=87=BA=E6=9D=A5=E7=9A=84=E6=9C=80=E5=90=8E=E5=87=A0=E4=BD=8D?= =?UTF-8?q?=EF=BC=88=E7=9B=B8=E5=AF=B9=201e-9=EF=BC=89=EF=BC=8C820.4=20?= =?UTF-8?q?=E5=AF=B9=20820=E3=80=81700.3=20=E5=AF=B9=20700=20=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E7=AE=97=E9=97=AD=E5=90=88=E3=80=82=E5=8F=AF=E5=8A=A0?= =?UTF-8?q?=E3=80=81=E6=AF=94=E7=8E=87=E4=B8=A4=E7=A7=8D=E4=B8=8B=E9=92=BB?= =?UTF-8?q?=E5=90=8C=E4=B8=80=E5=A5=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/common.py | 12 ++++++++---- backend/app/analysis/decompose.py | 4 ++-- backend/tests/test_analysis_decompose.py | 23 +++++++++++++++++++++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/backend/app/analysis/common.py b/backend/app/analysis/common.py index 3eb39865..b497e83f 100644 --- a/backend/app/analysis/common.py +++ b/backend/app/analysis/common.py @@ -107,22 +107,26 @@ def need(raw: Mapping[str, Any], key: str, what: str) -> Any: def written_places(raw: Any) -> int: - """快照里一格写到第几位小数(headline 是取整过的,按它的书写精度容差):数字文本按字面,浮点按最短写法,整数为 0。""" + """快照里一格写到第几位小数(headline 是取整过的,按它的书写精度容差):数字文本按字面,浮点按最短写法,整数为 0。是整数 + 的浮点(SQLite 的 ROUND() 交回 2120.0)是取到个位的,也为 0——最短写法里的「.0」不是书写精度。""" if isinstance(raw, bool) or raw is None: return 0 + if isinstance(raw, float) and raw.is_integer(): + return 0 text = repr(raw) if isinstance(raw, float) else str(raw).strip() if "e" in text.lower(): return 0 return len(text.split(".", 1)[1]) if "." in text else 0 -def closes(total: Fraction, cell: Fraction, raw: Any) -> bool: +def closes(total: Fraction, cell: Fraction, raw: Any, *, exact: bool = False) -> bool: """各段之和(或合起来的比率)和总数那一格对得上:十进制逐位相等;总数取整过的,差在它书写精度的半个末位以内也算;浮点 - 算出来的差在最后几位的也算(相对误差 1e-9 以内,同 engine/compare._equal)。""" + 算出来的差在最后几位的也算(相对误差 1e-9 以内,同 engine/compare._equal)。exact:对的是上一层那一段算出来的精确值 + (下钻),没有取整,只容浮点的最后几位。""" if total == cell: return True gap = abs(total - cell) - if gap <= Fraction(1, 2) / Fraction(10) ** written_places(raw): + if not exact and gap <= Fraction(1, 2) / Fraction(10) ** written_places(raw): return True scale = max(abs(total), abs(cell)) return scale != 0 and gap <= scale * Fraction(1, 10 ** 9) diff --git a/backend/app/analysis/decompose.py b/backend/app/analysis/decompose.py index 18520067..8d09e453 100644 --- a/backend/app/analysis/decompose.py +++ b/backend/app/analysis/decompose.py @@ -345,7 +345,7 @@ def _additive_result(env: Env, params: dict[str, Any], used: list[str], segs: li gaps[side] = heads[side][0] - total if parent_row is not None: expect = to_fraction(parent_row.get(side)) - if expect is None or not closes(total, expect, parent_row.get(side)): + if expect is None or not closes(total, expect, None, exact=True): gaps.setdefault(side, (expect or Fraction(0)) - total) closed = not gaps deltas = {x.key: x.target - x.base for x in segs} @@ -547,7 +547,7 @@ def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[ "d_target": sum((x.target[1] for x in live), Fraction(0))} for col, total in sums.items(): expect = to_fraction(parent_row.get(col)) - if expect is None or not closes(total, expect, parent_row.get(col)): + if expect is None or not closes(total, expect, None, exact=True): gaps.setdefault("base" if col.endswith("base") else "target", (expect or Fraction(0)) - total) closed = not gaps mix_total = sum((e[4] for e in effects), Fraction(0)) diff --git a/backend/tests/test_analysis_decompose.py b/backend/tests/test_analysis_decompose.py index f651c35d..7cb21a10 100644 --- a/backend/tests/test_analysis_decompose.py +++ b/backend/tests/test_analysis_decompose.py @@ -127,6 +127,29 @@ async def test_a_rounded_headline_closes_within_its_written_precision(): assert (result["verdict"]["code"] != "unclosed") is closes, written +async def test_a_headline_rounded_by_the_database_to_a_whole_number_closes_within_half_a_unit(): + # SQLite 的 ROUND() 交回浮点 2120.0:取到了个位,最短写法里的「.0」不是一位小数的书写精度 + rows = [["华东", "2026-W41", "1120.2"], ["华东", "2026-W42", "700"], ["华北", "2026-W41", "1000.2"], + ["华北", "2026-W42", "600"]] + for written, closes in ((2120.0, True), (2121.0, False), (2120.5, False)): + head = snap("Q1", "SELECT ROUND(SUM(amount)) AS gmv FROM orders", ["gmv_base", "gmv"], [[written, 1300.0]]) + result = await additive(rows, snaps=[weekly(rows), head], headline=["Q1.r0.gmv_base", "Q1.r0.gmv"]) + assert (result["verdict"]["code"] != "unclosed") is closes, written + + +async def test_a_drill_closes_against_the_exact_value_of_the_parent_segment(): + top = await additive() # 华东 820 → 700 + near = snap("Q4", "SELECT province, week, SUM(amount) AS gmv FROM orders WHERE region = '华东' GROUP BY 1, 2", + ["province", "week", "gmv"], + [["沪", "2026-W41", "500.4"], ["沪", "2026-W42", "410"], ["苏", "2026-W41", 320], + ["苏", "2026-W42", "290.3"]]) + params = {"query": "Q4", "by": ["province"], "period": "week", "base": "2026-W41", "target": "2026-W42", + "measure": "gmv", "parent": {"of": "A3", "key": ["华东"]}} + off = await analyze("decompose", "additive", snaps=[near], parents={"A3": top}, **params) + # 上一层那一段是算出来的精确值,不是取整过的总数:820.4 对 820、700.3 对 700 都不闭合 + assert off["verdict"]["code"] == "unclosed" and off["values"]["gap"] == "-0.4" + + async def test_drilling_down_checks_closure_against_the_parent_segment(): top = await additive() province = snap("Q4", "SELECT province, week, SUM(amount) AS gmv FROM orders WHERE region = '华东' GROUP BY 1, 2", From 472f78344d58e1744853f6511f362a8a81643c7e Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 16:00:29 +0800 Subject: [PATCH 08/13] =?UTF-8?q?=E5=88=86=E6=9E=90=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E8=A6=81=E6=B1=82=E3=80=8C=E9=9B=86=E4=B8=AD=E3=80=8D=E7=9A=84?= =?UTF-8?q?=E5=90=88=E8=AE=A1=E5=8D=A0=E6=AF=94=E4=B8=8D=E5=B0=8F=E4=BA=8E?= =?UTF-8?q?=E3=80=8C=E4=B8=BB=E8=A6=81=E6=9D=A5=E6=BA=90=E3=80=8D=E7=9A=84?= =?UTF-8?q?=E5=8D=A0=E6=AF=94=EF=BC=88=E5=A4=8D=E6=A0=B8=20R16=EF=BC=89?= =?UTF-8?q?=EF=BC=9Acontract.settings=5Fproblem=20=E5=8A=A0=E4=B8=80?= =?UTF-8?q?=E6=9D=A1=EF=BC=88=E6=B2=A1=E5=86=99=E7=9A=84=E9=A1=B9=E6=8C=89?= =?UTF-8?q?=E7=BC=BA=E7=9C=81=E8=A1=A5=E9=BD=90=E4=BA=86=E6=AF=94=EF=BC=89?= =?UTF-8?q?=EF=BC=8C=E8=AF=B4=E6=B3=95=E7=94=A8=20LABELS=20=E7=9A=84?= =?UTF-8?q?=E5=8F=AB=E6=B3=95=EF=BC=9A=E3=80=8C=E3=80=8C=E9=9B=86=E4=B8=AD?= =?UTF-8?q?=E3=80=8D=E7=9A=84=E5=90=88=E8=AE=A1=E5=8D=A0=E6=AF=94=E4=B8=8D?= =?UTF-8?q?=E8=83=BD=E5=B0=8F=E4=BA=8E=E3=80=8C=E4=B8=BB=E8=A6=81=E6=9D=A5?= =?UTF-8?q?=E6=BA=90=E3=80=8D=E7=9A=84=E5=8D=A0=E6=AF=94=E3=80=8D=E2=80=94?= =?UTF-8?q?=E2=80=94=E5=8E=9F=E6=9D=A5=E3=80=8C=E4=B8=BB=E8=A6=81=E6=9D=A5?= =?UTF-8?q?=E6=BA=90=E3=80=8D0.9=20=E9=85=8D=E3=80=8C=E9=9B=86=E4=B8=AD?= =?UTF-8?q?=E3=80=8D0.5=20=E8=83=BD=E9=80=9A=E8=BF=87=EF=BC=8C=E4=B8=80?= =?UTF-8?q?=E6=AE=B5=E5=8D=A0=2060%=20=E6=97=B6=E5=88=A4=E5=88=86=E6=95=A3?= =?UTF-8?q?=E3=80=81=E5=88=A4=E5=AE=9A=E8=AF=AD=E5=8D=B4=E8=AF=B4=E3=80=8C?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E5=93=AA=E4=B8=80=E6=AE=B5=E5=8D=A0=E5=88=B0?= =?UTF-8?q?=2090%=E3=80=8D=E3=80=82=E8=AE=BE=E7=BD=AE=E9=A1=B5=E5=86=99?= =?UTF-8?q?=E8=BF=9B=E6=9D=A5=E7=9A=84=E3=80=81=E5=88=86=E6=9E=90=E5=8D=A1?= =?UTF-8?q?=E4=B8=8A=E7=9A=84=E8=A6=86=E7=9B=96=E9=83=BD=E6=8C=89=E5=AE=83?= =?UTF-8?q?=E6=8C=A1=EF=BC=9Bapi/settings.analysis=5Fvalues=20=E8=AF=BB?= =?UTF-8?q?=E5=AD=98=E7=9D=80=E7=9A=84=E4=B8=80=E7=BB=84=E6=97=B6=E8=BF=99?= =?UTF-8?q?=E4=B8=80=E5=AF=B9=E5=AF=B9=E4=B8=8D=E4=B8=8A=E5=90=8C=E6=A0=B7?= =?UTF-8?q?=E4=B8=A4=E9=A1=B9=E9=83=BD=E6=8C=89=E7=BC=BA=E7=9C=81=EF=BC=88?= =?UTF-8?q?=E5=90=8C=E5=8F=A6=E5=A4=96=E4=B8=A4=E5=AF=B9=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/contract.py | 5 ++++- backend/app/api/settings.py | 5 +++-- backend/tests/test_analysis_contract.py | 4 ++++ backend/tests/test_analysis_settings.py | 3 +++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/app/analysis/contract.py b/backend/app/analysis/contract.py index 9159875c..d8d8c61a 100644 --- a/backend/app/analysis/contract.py +++ b/backend/app/analysis/contract.py @@ -167,7 +167,8 @@ LABELS: dict[str, str] = { def settings_problem(values: Mapping[str, Any]) -> str | None: """一组分析设置(或分析卡上的覆盖)哪里不对:不认识的键、类型不对、超出范围、k_alert 不大于 k_warn、 - max_history 小于 min_history。都对返回 None。说法给人看。""" + max_history 小于 min_history、concentration 小于 main_share(一段占了「主要」那么多还够不上「集中」,判定语会说「没有哪一段 + 占到」一个比它小的数)。没写的项按缺省补齐了比。都对返回 None。说法给人看(键的叫法取 LABELS)。""" for key, value in values.items(): if key not in RANGES: return f"不认识的分析设置:{key}" @@ -184,6 +185,8 @@ def settings_problem(values: Mapping[str, Any]) -> str | None: return f"{LABELS['k_alert']}要大于{LABELS['k_warn']}" if merged["max_history"] < merged["min_history"]: return f"{LABELS['max_history']}不能小于{LABELS['min_history']}" + if merged["concentration"] < merged["main_share"]: + return f"{LABELS['concentration']}不能小于{LABELS['main_share']}" return None diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index dda1f216..eca23947 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -907,13 +907,14 @@ def _analysis_item_ok(key: str, value: Any) -> bool: def analysis_values(stored: Any) -> dict[str, Any]: """库里存的(或刚写进来的)analysis 一组 → 补齐缺省;存坏了的项按缺省。两两之间的大小对不上的(「明显超出」不大于 - 「超出常规波动」、最多用几期历史小于至少几期)那一对都按缺省:只换掉一项,可能换出另一种对不上。""" + 「超出常规波动」、最多用几期历史小于至少几期、「集中」的合计占比小于「主要来源」的占比)那一对都按缺省:只换掉一项,可能 + 换出另一种对不上。""" stored = stored if isinstance(stored, dict) else {} out = dict(analysis_contract.DEFAULTS) for key in analysis_contract.DEFAULTS: if key in stored and _analysis_item_ok(key, stored[key]): out[key] = float(stored[key]) if analysis_contract.RANGES[key][0] is float else stored[key] - for pair in (("k_warn", "k_alert"), ("min_history", "max_history")): + for pair in (("k_warn", "k_alert"), ("min_history", "max_history"), ("main_share", "concentration")): if analysis_contract.settings_problem({k: out[k] for k in pair}) is not None: out.update({k: analysis_contract.DEFAULTS[k] for k in pair}) return out diff --git a/backend/tests/test_analysis_contract.py b/backend/tests/test_analysis_contract.py index cc1ea743..b72e240a 100644 --- a/backend/tests/test_analysis_contract.py +++ b/backend/tests/test_analysis_contract.py @@ -19,6 +19,10 @@ def test_settings_problems_are_spoken_with_labels_not_keys(): assert c.settings_problem({"main_share": True}) == "「主要来源」的占比要填数字" assert c.settings_problem({"main_share": 0.95}) == "「主要来源」的占比要在 0.3 到 0.9 之间" assert c.settings_problem({"max_history": 8, "min_history": 9}) == "最多用几期历史不能小于至少几期历史" + # 「主要」要的占比比「集中」的合计还高:一段占 60% 会判成分散,判定语却说「没有哪一段占到 90%」 + assert c.settings_problem({"main_share": 0.9, "concentration": 0.5}) == "「集中」的合计占比不能小于「主要来源」的占比" + assert c.settings_problem({"main_share": 0.85}) == "「集中」的合计占比不能小于「主要来源」的占比" # 缺省的 0.8 补齐了比 + assert c.settings_problem({"main_share": 0.8}) is None and c.settings_problem({"concentration": 0.5}) is None assert c.settings_problem({"nope": 1}) == "不认识的分析设置:nope" diff --git a/backend/tests/test_analysis_settings.py b/backend/tests/test_analysis_settings.py index 0b782c5a..1561a0af 100644 --- a/backend/tests/test_analysis_settings.py +++ b/backend/tests/test_analysis_settings.py @@ -69,6 +69,7 @@ def test_the_group_is_the_contract(): broken = {"main_share": "一半", "max_calls": 3.5, "k_warn": 4, "k_alert": 3.5, "min_history": 20, "max_history": 10, "table_top_k": 5, "trend_alpha": True} assert analysis_values(broken) == {**contract.DEFAULTS, "table_top_k": 5} + assert analysis_values({"main_share": 0.9, "concentration": 0.5}) == contract.DEFAULTS # 浮点的项存成浮点:发起时的快照、结论里记的阈值一个写法 assert analysis_values({"k_warn": 3, "k_alert": 4})["k_warn"] == 3.0 assert isinstance(analysis_values({"k_warn": 3, "k_alert": 4})["k_alert"], float) @@ -81,6 +82,8 @@ def test_the_group_is_the_contract(): ({"max_calls": 2.5}, f"{contract.LABELS['max_calls']}要填整数"), ({"k_warn": 3.5}, f"{contract.LABELS['k_alert']}要大于{contract.LABELS['k_warn']}"), ({"min_history": 60}, f"{contract.LABELS['max_history']}不能小于{contract.LABELS['min_history']}"), + ({"main_share": 0.9, "concentration": 0.5}, + f"{contract.LABELS['concentration']}不能小于{contract.LABELS['main_share']}"), ]) def test_problems_are_said_with_labels(value, why): assert why in (analysis_problem(value) or "") From fd8e392f489836d90199b2c773fc0b7e3fe8a0cf Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 16:02:44 +0800 Subject: [PATCH 09/13] =?UTF-8?q?=E7=BB=B4=E5=BA=A6=E5=88=97=E3=80=81?= =?UTF-8?q?=E6=9C=9F=E6=AC=A1=E5=88=97=E5=92=8C=E7=BB=93=E6=9E=9C=E8=A1=A8?= =?UTF-8?q?=E7=9A=84=E5=9B=BA=E5=AE=9A=E5=88=97=E9=87=8D=E5=90=8D=E6=97=B6?= =?UTF-8?q?=E6=8A=A5=E8=B0=83=E7=94=A8=E9=94=99=E8=AF=AF=EF=BC=88=E5=A4=8D?= =?UTF-8?q?=E6=A0=B8=20R17=EF=BC=89=EF=BC=9A=E7=BB=93=E6=9E=9C=E8=A1=A8?= =?UTF-8?q?=E7=9A=84=E5=88=97=20id=20=E6=98=AF=E8=BE=93=E5=85=A5=E7=9A=84?= =?UTF-8?q?=E7=BB=B4=E5=BA=A6=E5=88=97=EF=BC=88=E6=9C=9F=E6=AC=A1=E5=88=97?= =?UTF-8?q?=EF=BC=89=E5=8A=A0=E4=B8=8A=E5=9B=BA=E5=AE=9A=E7=9A=84=E5=87=A0?= =?UTF-8?q?=E5=88=97=EF=BC=8C=E7=BB=B4=E5=BA=A6=E5=88=97=E5=8F=AB=20flag?= =?UTF-8?q?=E3=80=81base=E3=80=81target=E3=80=81delta=E3=80=81share?= =?UTF-8?q?=EF=BC=88=E5=8F=AF=E5=8A=A0=EF=BC=89=E6=88=96=20mix=E3=80=81rat?= =?UTF-8?q?e=E3=80=81total=E3=80=81w=5Fbase=E2=80=A6=E2=80=A6=EF=BC=88?= =?UTF-8?q?=E6=AF=94=E7=8E=87=EF=BC=89=EF=BC=8C=E6=9C=9F=E6=AC=A1=E5=88=97?= =?UTF-8?q?=E5=8F=AB=20value=E3=80=81change=E3=80=81role=EF=BC=88=E6=B3=A2?= =?UTF-8?q?=E5=8A=A8=E5=9F=BA=E7=BA=BF=E3=80=81=E8=B6=8B=E5=8A=BF=EF=BC=89?= =?UTF-8?q?=E6=97=B6=E5=88=97=20id=20=E9=87=8D=E5=A4=8D=EF=BC=8C=E5=BC=95?= =?UTF-8?q?=E7=94=A8=E7=BB=93=E6=9E=9C=E8=A1=A8=E7=9A=84=E6=A0=BC=E4=BC=9A?= =?UTF-8?q?=E8=AE=A4=E9=94=99=EF=BC=8C=E4=B8=8B=E9=92=BB=E8=BF=98=E4=BC=9A?= =?UTF-8?q?=E6=8A=A5=E3=80=8CA1=20=E6=8C=89=200=20=E5=88=97=E6=8B=86?= =?UTF-8?q?=E3=80=8D=E3=80=82common.reserved=5Fcheck=20=E4=B8=8D=E5=88=86?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F=E5=86=99=E6=8C=A1=E4=BD=8F=E8=BF=99=E4=BA=9B?= =?UTF-8?q?=E5=90=8D=E5=AD=97=EF=BC=8C=E5=91=8A=E8=AF=89=20agent=20?= =?UTF-8?q?=E5=9C=A8=20SQL=20=E9=87=8C=E6=8D=A2=E4=B8=AA=E5=88=AB=E5=90=8D?= =?UTF-8?q?=EF=BC=88=E6=AF=94=E5=A6=82=20AS=20flag=5Fname=EF=BC=89?= =?UTF-8?q?=EF=BC=9B=E8=BF=99=E6=98=AF=E7=BB=93=E6=9E=84=E6=80=A7=E7=9A=84?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E9=94=99=E8=AF=AF=EF=BC=8C=E5=88=86=E6=9E=90?= =?UTF-8?q?=E5=8D=A1=E9=87=8C=E7=85=A7=E6=97=A7=E8=AE=A9=E5=8D=A1=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/common.py | 10 +++++++++- backend/app/analysis/decompose.py | 18 +++++++++++------- backend/app/analysis/series.py | 8 ++++++-- backend/tests/test_analysis_decompose.py | 17 +++++++++++++++++ backend/tests/test_analysis_series.py | 8 ++++++++ 5 files changed, 51 insertions(+), 10 deletions(-) diff --git a/backend/app/analysis/common.py b/backend/app/analysis/common.py index b497e83f..3c278c5b 100644 --- a/backend/app/analysis/common.py +++ b/backend/app/analysis/common.py @@ -106,6 +106,14 @@ def need(raw: Mapping[str, Any], key: str, what: str) -> Any: return value +def reserved_check(column: str, reserved: Sequence[str], what: str) -> None: + """结果表的列 id 是输入的维度列(期次列)加上固定的几列:重名了列 id 就重复,引用结果表的格、下钻按的维度数都会认错。 + 遇到这些名字(不分大小写)是调用错误,让 agent 在 SQL 里换个别名。""" + if column.lower() in {r.lower() for r in reserved}: + raise CallError("reserved_column", f"{what}「{column}」和结果表的列重名(结果表要用 {'、'.join(reserved)} 这几个名字):" + f"在 SQL 里给它换个别名再分析,比如 AS {column}_name") + + def written_places(raw: Any) -> int: """快照里一格写到第几位小数(headline 是取整过的,按它的书写精度容差):数字文本按字面,浮点按最短写法,整数为 0。是整数 的浮点(SQLite 的 ROUND() 交回 2120.0)是取到个位的,也为 0——最短写法里的「.0」不是书写精度。""" @@ -152,4 +160,4 @@ def sign_word(value: Fraction | float | int) -> str | None: __all__ = ["DERIVED_PARAMS", "Env", "build", "closes", "column_label", "column_types", "label_for", "need", "refuse", - "sign_word", "today_param", "written_places"] + "reserved_check", "sign_word", "today_param", "written_places"] diff --git a/backend/app/analysis/decompose.py b/backend/app/analysis/decompose.py index 8d09e453..1efc4291 100644 --- a/backend/app/analysis/decompose.py +++ b/backend/app/analysis/decompose.py @@ -26,8 +26,8 @@ from typing import Any from app.analysis import comparable as cmp from app.analysis import phrases as ph -from app.analysis.common import (Env, build, closes, column_label, column_types, label_for, need, refuse, sign_word, - today_param) +from app.analysis.common import (Env, build, closes, column_label, column_types, label_for, need, refuse, + reserved_check, sign_word, today_param) from app.analysis.contract import CallError from app.analysis.inputs import Snap, as_list, hashable, same_value from app.analysis.numbers import canon, to_fraction @@ -81,9 +81,10 @@ def _key_strings(key: Sequence[Any]) -> list[str]: return [ph.key_text([k]) for k in key] -def _sides(raw: Mapping[str, Any], env: Env, roles: Sequence[tuple[str, str]], *, by_needed: bool = True - ) -> tuple[dict[str, Any], Snap, Snap, dict[str, str], list[str]]: - """一份或两份快照:(规范化的参数, 基期快照, 本期快照, {角色: 列名}, 维度列)。roles 是 [(参数名, 角色的叫法)]。""" +def _sides(raw: Mapping[str, Any], env: Env, roles: Sequence[tuple[str, str]], *, by_needed: bool = True, + reserved: Sequence[str] = ()) -> tuple[dict[str, Any], Snap, Snap, dict[str, str], list[str]]: + """一份或两份快照:(规范化的参数, 基期快照, 本期快照, {角色: 列名}, 维度列)。roles 是 [(参数名, 角色的叫法)];reserved 是 + 结果表固定的列名,维度列不能叫这些。""" params: dict[str, Any] = {} two = raw.get("base_query") is not None or raw.get("target_query") is not None if two: @@ -105,6 +106,8 @@ def _sides(raw: Mapping[str, Any], env: Env, roles: Sequence[tuple[str, str]], * if two: for c in by: base.column(c, "维度列") + for c in by: + reserved_check(c, reserved, "维度列") params["by"] = by cols: dict[str, str] = {} for key, what in roles: @@ -284,7 +287,7 @@ def _share_rank(segments: Sequence[Any], share: Any) -> list[Any]: def additive(raw: Mapping[str, Any], env: Env) -> Any: used = list(USED["additive"]) - params, bs, ts, cols, by = _sides(raw, env, [("measure", "度量列")]) + params, bs, ts, cols, by = _sides(raw, env, [("measure", "度量列")], reserved=_FIXED_ADDITIVE) measure = cols["measure"] label = label_for(env, raw.get("label"), ts, measure) if raw.get("label") is not None: @@ -444,7 +447,8 @@ def _additive_result(env: Env, params: dict[str, Any], used: list[str], segs: li def ratio(raw: Mapping[str, Any], env: Env) -> Any: used = list(USED["ratio"]) - params, bs, ts, cols, by = _sides(raw, env, [("numerator", "分子列"), ("denominator", "分母列")]) + params, bs, ts, cols, by = _sides(raw, env, [("numerator", "分子列"), ("denominator", "分母列")], + reserved=_FIXED_RATIO) num, den = cols["numerator"], cols["denominator"] num_label, den_label = column_label(env, ts, num), column_label(env, ts, den) label = ph.pick_label(raw.get("label"), ph.say(num_label, "与", den_label, "之比")) diff --git a/backend/app/analysis/series.py b/backend/app/analysis/series.py index 840f4d56..c9b57687 100644 --- a/backend/app/analysis/series.py +++ b/backend/app/analysis/series.py @@ -31,7 +31,8 @@ from app.analysis import comparable as cmp from app.analysis import periods as pd from app.analysis import phrases as ph from app.analysis import stats -from app.analysis.common import Env, build, column_label, column_types, label_for, need, refuse, sign_word, today_param +from app.analysis.common import (Env, build, column_label, column_types, label_for, need, refuse, reserved_check, + sign_word, today_param) from app.analysis.contract import CallError from app.analysis.inputs import Snap, same_value from app.analysis.numbers import canon, to_fraction @@ -45,6 +46,8 @@ USED: dict[str, tuple[str, ...]] = { "baseline": ("k_warn", "k_alert", "min_history", "max_history"), "trend": ("trend_min_points", "trend_alpha", "max_history"), } +#: 结果表在期次列之外固定的几列(期次列不能叫这些) +FIXED: dict[str, tuple[str, ...]] = {"baseline": ("value", "change", "role"), "trend": ("value", "change")} def refusal_basis(op: str, settings: Mapping[str, Any]) -> tuple[list[str], str]: @@ -83,6 +86,7 @@ def _load(raw: Mapping[str, Any], env: Env, used: list[str]) -> Series | Any: """读出一条序列;算不了时直接交回 refused 的结论。""" snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号") period = snap.column(need(raw, "period", "期次列"), "期次列") + reserved_check(period, FIXED[env.op], "期次列") measure = snap.column(need(raw, "measure", "度量列"), "度量列") grain = raw.get("grain") if grain is not None and grain not in (*pd.GRAINS, "seq"): @@ -369,4 +373,4 @@ def _tail_gap(series: Series, ordinal: int) -> str: return "再往前缺期" -__all__ = ["MAD_SCALE", "USED", "baseline", "refusal_basis", "run_length", "trend", "unreadable_text"] +__all__ = ["FIXED", "MAD_SCALE", "USED", "baseline", "refusal_basis", "run_length", "trend", "unreadable_text"] diff --git a/backend/tests/test_analysis_decompose.py b/backend/tests/test_analysis_decompose.py index 7cb21a10..67ab1946 100644 --- a/backend/tests/test_analysis_decompose.py +++ b/backend/tests/test_analysis_decompose.py @@ -9,6 +9,7 @@ from fractions import Fraction import pytest +from app.analysis import DATA_ERRORS from app.analysis.decompose import lmdi, ratio_effects from tests.analysis_samples import analyze, call_error, fixed_prober, notes, snap @@ -487,3 +488,19 @@ async def test_a_label_cannot_carry_a_judgment_into_the_phrase(): # 消过一遍会变的叫法:复算拿记下的叫法再消,判定语逐字相同(analyze 核对复算逐字段相同) marked = await additive(label="销售额[ [v:Q1.r0.gmv] ]") assert marked["params"]["label"] == "销售额" and marked["phrase"].startswith("销售额下降 194") + + +async def test_a_dimension_named_like_a_result_column_is_a_call_error(): + rows = [["线上", "2026-W41", 820], ["线上", "2026-W42", 700], ["线下", "2026-W41", 600], ["线下", "2026-W42", 560]] + flagged = snap("Q2", "SELECT channel AS flag, week, SUM(amount) AS gmv FROM orders GROUP BY 1, 2", + ["flag", "week", "gmv"], rows) + error = await call_error("decompose", "additive", snaps=[flagged], query="Q2", by=["flag"], period="week", + base="2026-W41", target="2026-W42", measure="gmv") + assert error.code == "reserved_column" and "在 SQL 里给它换个别名" in error.message and "AS flag_name" in error.message + assert error.code not in DATA_ERRORS # 结构性的:分析卡里照旧让卡失败 + ratio_rows = [["线上", "W41", 50, 1000], ["线上", "W42", 80, 1000], ["线下", "W41", 20, 1000], ["线下", "W42", 26, 1000]] + mixed = snap("Q1", "SELECT channel AS Mix, week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders " + "GROUP BY 1, 2", ["Mix", "week", "refunds", "orders"], ratio_rows) + error = await call_error("decompose", "ratio", snaps=[mixed], query="Q1", by="Mix", period="week", base="W41", + target="W42", numerator="refunds", denominator="orders") + assert error.code == "reserved_column" and "「Mix」" in error.message diff --git a/backend/tests/test_analysis_series.py b/backend/tests/test_analysis_series.py index f52ae0c6..d88e31dd 100644 --- a/backend/tests/test_analysis_series.py +++ b/backend/tests/test_analysis_series.py @@ -319,3 +319,11 @@ async def test_a_falling_trend_whose_last_weeks_rise_says_but(): falling = await trend(weeks([200, 196, 190, 188, 181, 176, 170, 165, 160, 162, 165, 168])) assert falling["verdict"]["code"] == "falling" and falling["values"]["run_direction"] == "up" assert falling["phrase"].endswith(",但最近已连续 3 周上升") + + +async def test_a_period_column_named_like_a_result_column_is_a_call_error(): + rows = [[r[0], r[1]] for r in weeks(NOISE + [101])] + named = snap("Q1", "SELECT DATE(order_date, 'weekday 0', '-6 days') AS value, COUNT(*) AS orders FROM orders " + "GROUP BY 1", ["value", "orders"], rows) + error = await call_error("baseline", snaps=[named], query="Q1", period="value", measure="orders") + assert error.code == "reserved_column" and "期次列「value」和结果表的列重名" in error.message From f7a09eb108ec920c489e4f2fa4693541e832e4dc Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 16:09:21 +0800 Subject: [PATCH 10/13] =?UTF-8?q?=E4=B8=80=E4=BB=BD=E5=BF=AB=E7=85=A7?= =?UTF-8?q?=E6=8C=89=E6=9C=9F=E6=AC=A1=E5=88=86=E7=BB=84=E7=9A=84=E5=8F=98?= =?UTF-8?q?=E5=8C=96=E6=8B=86=E8=A7=A3=E4=B9=9F=E6=A0=B8=E6=9C=AC=E6=9C=9F?= =?UTF-8?q?=E9=BD=90=E4=B8=8D=E9=BD=90=EF=BC=88=E5=A4=8D=E6=A0=B8=20R20?= =?UTF-8?q?=EF=BC=89=EF=BC=9A=E5=8F=AF=E5=8A=A0=E3=80=81=E6=AF=94=E7=8E=87?= =?UTF-8?q?=E3=80=81=E4=B9=98=E7=A7=AF=E4=B8=89=E7=A7=8D=E6=A8=A1=E5=BC=8F?= =?UTF-8?q?=E7=94=A8=E6=B3=A2=E5=8A=A8=E5=9F=BA=E7=BA=BF=E5=90=8C=E4=B8=80?= =?UTF-8?q?=E5=A5=97=E5=AE=8C=E6=95=B4=E6=80=A7=E6=A3=80=E6=9F=A5=EF=BC=88?= =?UTF-8?q?comparable.period=5Fcomplete=EF=BC=9A=E4=B8=8D=E5=88=B0?= =?UTF-8?q?=E6=95=B4=E5=91=A8=E3=80=81=E6=B2=A1=E8=BF=87=E5=AE=8C=E3=80=81?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=B2=A1=E5=88=B0=E3=80=81SQL=20=E6=88=AA?= =?UTF-8?q?=E6=8E=89=E4=B8=80=E9=83=A8=E5=88=86=EF=BC=89=EF=BC=8C=E6=9C=AC?= =?UTF-8?q?=E6=9C=9F=E4=B8=8D=E9=BD=90=E7=AE=97=E4=B8=8D=E4=BA=86=EF=BC=88?= =?UTF-8?q?partial=5Ftarget=EF=BC=8C=E3=80=8C=E6=9C=AC=E6=9C=9F=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=8F=AA=E5=88=B0=202026-10-20=EF=BC=8C=E8=BF=98?= =?UTF-8?q?=E5=B7=AE=205=20=E5=A4=A9=EF=BC=8C=E5=92=8C=E5=AE=8C=E6=95=B4?= =?UTF-8?q?=E7=9A=84=E5=91=A8=E6=AF=94=E6=B2=A1=E6=9C=89=E6=84=8F=E4=B9=89?= =?UTF-8?q?=E3=80=8D=EF=BC=89=EF=BC=8C=E6=A0=B8=E5=AF=B9=E4=B8=8D=E4=BA=86?= =?UTF-8?q?=EF=BC=88=E6=9F=A5=E4=B8=8D=E5=88=B0=E8=A1=A8=E7=9A=84=E8=8C=83?= =?UTF-8?q?=E5=9B=B4=E3=80=81=E5=91=A8=E5=8F=B7=E8=AE=A4=E4=B8=8D=E5=87=BA?= =?UTF-8?q?=E7=AE=97=E6=B3=95=E3=80=81=E6=9C=9F=E6=AC=A1=E4=B8=8D=E6=98=AF?= =?UTF-8?q?=E6=97=A5=E6=9C=9F=EF=BC=89=E9=99=84=E4=B8=80=E5=8F=A5=E8=AF=B4?= =?UTF-8?q?=E6=98=8E=EF=BC=8C=E5=8F=AA=E5=9C=A8=E8=AF=81=E6=8D=AE=E9=9D=A2?= =?UTF-8?q?=E6=9D=BF=E3=80=81=E4=B8=8D=E8=BF=9B=E5=88=A4=E5=AE=9A=E8=AF=AD?= =?UTF-8?q?=EF=BC=88=E4=B8=A4=E4=BB=BD=E5=BF=AB=E7=85=A7=E6=97=B6=E5=8F=AF?= =?UTF-8?q?=E6=AF=94=E6=80=A7=E7=BB=93=E8=AE=BA=E7=85=A7=E6=97=A7=E8=BF=9B?= =?UTF-8?q?=E5=88=A4=E5=AE=9A=E8=AF=AD=EF=BC=89=EF=BC=9B=E7=BB=93=E8=AE=BA?= =?UTF-8?q?=E7=9A=84=20params=20=E8=AE=B0=E4=B8=8B=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E5=BD=93=E5=A4=A9=E3=80=82=E6=9C=9F=E6=AC=A1=E6=8C=89=20period?= =?UTF-8?q?s.parse=5Fcompared=20=E8=AE=A4=EF=BC=9A=E5=91=A8=E5=8F=B7?= =?UTF-8?q?=E6=8C=89=20SQL=20=E8=AE=A4=E7=AE=97=E6=B3=95=EF=BC=9B=E5=BF=AB?= =?UTF-8?q?=E7=85=A7=E9=87=8C=E5=BE=80=E5=BE=80=E5=8F=AA=E6=9C=89=E6=AF=94?= =?UTF-8?q?=E7=9A=84=E4=B8=A4=E6=9C=9F=EF=BC=8C=E5=80=BC=E9=83=BD=E6=98=AF?= =?UTF-8?q?=E6=97=A5=E6=9C=9F=E3=80=81=E6=8E=A8=E6=96=AD=E6=88=90=E6=8C=89?= =?UTF-8?q?=E5=91=A8=E3=80=81=E6=9C=9F=E6=AC=A1=E5=88=97=E5=8F=88=E6=98=AF?= =?UTF-8?q?=E7=9B=B4=E6=8E=A5=E5=8F=96=E7=9A=84=E6=97=A5=E6=9C=9F=E5=88=97?= =?UTF-8?q?=EF=BC=88=E5=8E=9F=E6=A0=B7=E3=80=81date(=E5=88=97)=E3=80=81DAT?= =?UTF-8?q?E(=E5=88=97)=E3=80=81CAST=20=E6=88=90=E6=97=A5=E6=9C=9F?= =?UTF-8?q?=EF=BC=89=E6=97=B6=E6=8C=89=E5=A4=A9=E8=AE=A4=E2=80=94=E2=80=94?= =?UTF-8?q?=E6=98=A8=E5=A4=A9=E5=AF=B9=E4=B8=8A=E5=91=A8=E5=90=8C=E4=B8=80?= =?UTF-8?q?=E5=A4=A9=E6=AD=A3=E5=A5=BD=E9=9A=94=207=20=E5=A4=A9=EF=BC=8C?= =?UTF-8?q?=E5=85=89=E7=9C=8B=E5=80=BC=E5=88=86=E4=B8=8D=E5=87=BA=E6=98=AF?= =?UTF-8?q?=E4=B8=A4=E5=A4=A9=E8=BF=98=E6=98=AF=E4=B8=A4=E5=91=A8=E3=80=82?= =?UTF-8?q?=E5=8F=AF=E6=AF=94=E6=80=A7=E6=A3=80=E6=9F=A5=E6=8C=89=E6=9C=9F?= =?UTF-8?q?=E6=AC=A1=E5=88=86=E7=BB=84=E3=80=81=E6=B2=A1=E7=BB=99=E7=B2=92?= =?UTF-8?q?=E5=BA=A6=E6=97=B6=E5=90=8C=E6=A0=B7=E6=8C=89=E5=AE=83=E8=AE=A4?= =?UTF-8?q?=E3=80=82=E8=AF=81=E6=8D=AE=E6=8E=A5=E5=8F=A3=E7=9A=84=E4=B8=80?= =?UTF-8?q?=E6=9D=A1=E6=B5=8B=E8=AF=95=EF=BC=88tests/test=5Fanalysis=5Fevi?= =?UTF-8?q?dence=5Fapi.py=EF=BC=89=E8=B7=9F=E7=9D=80=E8=A1=A5=E4=B8=8A?= =?UTF-8?q?=E8=BF=99=E4=B8=80=E5=8F=A5=E8=AF=B4=E6=98=8E=E7=9A=84=E7=A7=8D?= =?UTF-8?q?=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/comparable.py | 3 +- backend/app/analysis/decompose.py | 62 ++++++++++++++++++--- backend/app/analysis/periods.py | 31 ++++++++++- backend/tests/test_analysis_decompose.py | 58 ++++++++++++++++++- backend/tests/test_analysis_evidence_api.py | 2 +- backend/tests/test_analysis_periods.py | 12 ++++ 6 files changed, 155 insertions(+), 13 deletions(-) diff --git a/backend/app/analysis/comparable.py b/backend/app/analysis/comparable.py index bc157432..bc428e89 100644 --- a/backend/app/analysis/comparable.py +++ b/backend/app/analysis/comparable.py @@ -568,7 +568,8 @@ def comparable(raw: Mapping[str, Any], env: Any) -> Any: "里有的期次值", reason=f"期次列「{period}」里没有{what}「{ph.key_text([params[side]])}」") env.inputs.use(target, "data", [period, *([measure] if measure else [])], rows["base"] + rows["target"]) - parsed = pd.parse(values, grain, weeks=pd.week_scheme(target.sql, period, target.dialect)) + parsed = pd.parse(values, grain, weeks=pd.week_scheme(target.sql, period, target.dialect)) if grain \ + else pd.parse_compared(values, target.sql, period, target.dialect) if parsed is None: item_b = item_t = pd.Period(None, 0, None, None) grain = "seq" diff --git a/backend/app/analysis/decompose.py b/backend/app/analysis/decompose.py index 1efc4291..104001b6 100644 --- a/backend/app/analysis/decompose.py +++ b/backend/app/analysis/decompose.py @@ -2,9 +2,10 @@ **参数**(AnalysisRequest.params;列名不分大小写,按快照认): -- 一份快照:query(编号)、period(期次列)、base / target(期次列里的两个值);或两份快照:base_query、target_query(各自 - 只有维度列和度量列,先做可比性检查:口径结构不同的不拆,算不了 incomparable_sides;其余不可比的照样算,判定后附上 - 可比性结论)。 +- 一份快照:query(编号)、period(期次列)、base / target(期次列里的两个值),本期齐不齐同波动基线那一套(comparable. + period_complete:不到整期、没过完、数据没到 → partial_target;核对不了的附说明,只在证据面板);或两份快照:base_query、 + target_query(各自只有维度列和度量列,先做可比性检查:口径结构不同的不拆,算不了 incomparable_sides;其余不可比的照样算, + 判定后附上可比性结论)。 - additive:by(维度列,1–2 列)、measure(度量列);可选 headline {base, target}(基期、本期总数各一格,如 Q1.r1.gmv,用来 核对闭合)、parent {of: A3, key: [华东]}(下钻:这一层各段之和要等于 A3 里华东那一段)。 - ratio:by、numerator(分子列)、denominator(分母列);可选 headline {base, target}(整体比率的两格)、parent(上一层的比率 @@ -275,6 +276,30 @@ def _comparability(env: Env, params: dict[str, Any], base: Snap, target: Snap, m return outcome, outcome.structure +def _period_complete(env: Env, params: dict[str, Any], snap: Snap, measure: str | None + ) -> tuple[str | None, dict[str, str] | None]: + """一份快照按期次分组时本期齐不齐(同波动基线那一套,comparable.period_complete;期次按 periods.parse_compared 认): + (不齐时 partial_target 的原因, 核对不了时的附带说明)。两份快照的在可比性检查里核过整期,不在这里核。""" + from app.analysis import periods as pd + + if "base_query" in params: + return None, None + params["today"] = today_param(env) + period = params["period"] + i = snap.index(period) + values = [row[i] if i < len(row) else None for row in snap.rows] + parsed = pd.parse_compared(values, snap.sql, period, snap.dialect) + if parsed is None: + return None, ph.caveat("comparability", ph.NO_DATES) + items, grain = parsed + target = next(item for item, v in zip(items, values) if same_value(params["target"], v)) + status, why, note = cmp.period_complete(snap, period, target, grain, measure, env.facts, env.book, + env.today) # type: ignore[arg-type] + if status == "short": + return f"{why},和完整的{pd.PER_WORD[grain]}比没有意义", None + return None, ph.caveat("comparability", note) if status == "unknown" and note else None + + def _share_rank(segments: Sequence[Any], share: Any) -> list[Any]: """按 share 从大到小(一样大的按先出现的)。""" return sorted(segments, key=lambda s: -share(s)) @@ -323,6 +348,12 @@ def additive(raw: Mapping[str, Any], env: Env) -> Any: if structure is not None: return refuse(env, params, used, "incomparable_sides", f"{structure},不是同一个量的两期,不拆", criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats) + partial, note = _period_complete(env, params, ts, measure) + if partial is not None: + return refuse(env, params, used, "partial_target", partial, + criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats) + if note is not None: + caveats.append(note) segs = _segments(side_b, side_t, Fraction(0)) if len(segs) < 2: return refuse(env, params, used, "too_few_segments", @@ -435,7 +466,9 @@ def _additive_result(env: Env, params: dict[str, Any], used: list[str], segs: li values["gap"] = canon(max(gaps.values(), key=abs)) verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects], "direction": sign_word(delta) if code in ("driver", "concentrated", "dispersed") else None} - phrase = ph.with_caveats(main, caveats, skip=("offset",) if code == "flat" else ()) + # 一份快照时 comparability 只会是本期核对不了的那句(同波动基线,只在证据面板);两份快照的可比性结论照旧进判定语 + skip = (("offset",) if code == "flat" else ()) + (("comparability",) if outcome is None else ()) + phrase = ph.with_caveats(main, caveats, skip=skip) return build(env, params, used, table=table, values=values, verdict=verdict, criterion=ph.criterion_additive(s, top_k_max=k_max, summed=summed), phrase=phrase, caveats=caveats) @@ -487,6 +520,9 @@ def ratio(raw: Mapping[str, Any], env: Env) -> Any: outcome, structure = _comparability(env, params, bs, ts, den) if structure is not None: return refuse(env, params, used, "incomparable_sides", f"{structure},不是同一个量的两期,不拆", criterion=criterion) + partial, note = _period_complete(env, params, ts, den) + if partial is not None: + return refuse(env, params, used, "partial_target", partial, criterion=criterion) segs = _segments(side_b, side_t, (Fraction(0), Fraction(0))) live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0] if len(live) < 2: @@ -504,7 +540,7 @@ def ratio(raw: Mapping[str, Any], env: Env) -> Any: ph.say(ph.key_text(x.key), f"{when}的", den_label, "为 0 而", num_label, "不为 0,比率拆不了"), criterion=criterion) return _ratio_result(env, params, used, segs, by, dims, label, num_label, den_label, heads, parent_row, outcome, - criterion, target_snap=ts) + criterion, target_snap=ts, note=note) def ratio_effects(segs: Sequence[tuple[Fraction, Fraction, Fraction, Fraction]] @@ -533,7 +569,8 @@ def ratio_effects(segs: Sequence[tuple[Fraction, Fraction, Fraction, Fraction]] def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[Segment], by: list[str], dims: list[str], label: str, num_label: str, den_label: str, heads: Mapping[str, tuple[Fraction, Any]], parent_row: Mapping[str, Any] | None, - outcome: cmp.Outcome | None, criterion: str, *, target_snap: Snap) -> Any: + outcome: cmp.Outcome | None, criterion: str, *, target_snap: Snap, + note: Mapping[str, str] | None = None) -> Any: main_share, flat_pct = env.threshold("main_share"), env.threshold("flat_pct") min_den = env.threshold("min_denominator") live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0] @@ -595,6 +632,8 @@ def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[ ph.say(num_label, "或", den_label)))) if outcome is not None and outcome.verdict != "comparable": caveats.append(ph.caveat("comparability", outcome.phrase)) + if note is not None: + caveats.append(dict(note)) total_text = ph.ratio_change(R, R2, proportion=proportion) if code == "mix_driven": star = subjects[0] @@ -641,7 +680,8 @@ def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[ verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects], "direction": sign_word(delta) if code in ("mix_driven", "rate_driven", "both") else None} return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, - phrase=ph.with_caveats(main, caveats), caveats=caveats) + phrase=ph.with_caveats(main, caveats, skip=("comparability",) if outcome is None else ()), + caveats=caveats) # -------------------------------------------------------------------------- @@ -715,6 +755,9 @@ def factor(raw: Mapping[str, Any], env: Env) -> Any: outcome, structure = _comparability(env, params, bs, ts, product) if structure is not None: return refuse(env, params, used, "incomparable_sides", f"{structure},不是同一个量的两期,不拆", criterion=criterion) + partial, note = _period_complete(env, params, ts, product) + if partial is not None: + return refuse(env, params, used, "partial_target", partial, criterion=criterion) if any(v <= 0 for row in cells.values() for v, _ in row): return refuse(env, params, used, "non_positive", "乘积或因子里有不大于 0 的值,对数拆解要求全为正", criterion=criterion) for when, row in cells.items(): @@ -738,6 +781,8 @@ def factor(raw: Mapping[str, Any], env: Env) -> Any: caveats: list[dict[str, str]] = [] if outcome is not None and outcome.verdict != "comparable": caveats.append(ph.caveat("comparability", outcome.phrase)) + if note is not None: + caveats.append(note) total_text = ph.change(delta, M) top = max(range(len(factors)), key=lambda i: shares[i] if shares[i] is not None else -math.inf) if flat: @@ -765,7 +810,8 @@ def factor(raw: Mapping[str, Any], env: Env) -> Any: "top_share": canon(shares[top]) if shares[top] is not None else None} verdict = {"code": code, "subjects": subjects, "direction": sign_word(delta) if code != "flat" else None} return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, - phrase=ph.with_caveats(main, caveats), caveats=caveats) + phrase=ph.with_caveats(main, caveats, skip=("comparability",) if outcome is None else ()), + caveats=caveats) __all__ = ["HEADLINE_SHAPE", "PARENT_SHAPE", "SAME_QUERY", "SIMPSON_SHARE", "USED", "additive", "factor", "lmdi", diff --git a/backend/app/analysis/periods.py b/backend/app/analysis/periods.py index 8082d248..63e6fd28 100644 --- a/backend/app/analysis/periods.py +++ b/backend/app/analysis/periods.py @@ -271,6 +271,35 @@ def week_scheme(sql: str, column: str, dialect: str | None) -> str | None: return None +@lru_cache(maxsize=256) +def plain_day(sql: str, column: str, dialect: str | None) -> bool: + """期次列是不是直接取的一个日期列(原样、date(列)、DATE(列)、CAST(列 AS DATE)),没有按周、按月换算过。""" + from sqlglot import exp + + from app.engine import cell_format as cf + + try: + value = _period_expr(sql, column, dialect) + except Exception: # noqa: BLE001 + return False + if isinstance(value, (exp.Date, exp.TsOrDsToDate)): + extra = [v for k, v in value.args.items() if k != "this" and v not in (None, [], False)] + value = cf._unwrap(value.this) if not extra else None + return isinstance(value, exp.Column) and not isinstance(value.this, exp.Star) + + +def parse_compared(values: Sequence[Any], sql: str, column: str, + dialect: str | None) -> tuple[list[Period], str] | None: + """按期次分组的一份快照里比的两期(变化拆解、可比性检查没给粒度时;快照里往往只有这两期):同 parse,周号按 SQL 认算法, + 粒度按值推断;但值都是日期、推断成按周、期次列又是直接取的日期列(plain_day)时按天认——两期正好隔 7 天,光看值分不出 + 是两天还是两周,期次列没按周换算过就是那一天。""" + parsed = parse(values, weeks=week_scheme(sql, column, dialect)) + if parsed is not None and parsed[1] == "week" and all(_shape(v) == "date" for v in values) \ + and plain_day(sql, column, dialect): + return parse(values, "day") + return parsed + + def _period_expr(sql: str, column: str, dialect: str | None) -> Any: """结果列 column 的写法(sqlglot 节点):最外层 SELECT 里那一项;是从 CTE、子查询里原样取出的一列时往里找。""" from sqlglot import exp @@ -497,4 +526,4 @@ def weekend_days(start: date, end: date) -> int: __all__ = ["COUNT_WORD", "DEFAULT_LAG", "GRAINS", "PER_WORD", "WEEK_SCHEMES", "Period", "calendar_unit", "parse", - "short_days", "unreadable", "week_scheme", "week_span", "weekend_days"] + "parse_compared", "plain_day", "short_days", "unreadable", "week_scheme", "week_span", "weekend_days"] diff --git a/backend/tests/test_analysis_decompose.py b/backend/tests/test_analysis_decompose.py index 67ab1946..00024921 100644 --- a/backend/tests/test_analysis_decompose.py +++ b/backend/tests/test_analysis_decompose.py @@ -57,8 +57,10 @@ async def test_a_driver_is_named_with_its_share_and_the_whole_shape_is_filled(): "columns": ["region", "week", "gmv"], "rows": "all"}] assert result["hypothesis"] == "华东拖累" and result["probes"] == [] assert result["criterion"].endswith("以各段之和为总数") - assert {c["code"] for c in result["caveats"]} == {"headline_basis"} + # 期次是现成的一列「2026-W41」,认不出是按哪种周号算的:本期齐不齐核对不了,只附在证据面板(不进判定语) + assert {c["code"] for c in result["caveats"]} == {"headline_basis", "comparability"} assert result["params"]["catalog"] == {"labels": {"region": "地区"}, "measures": {"gmv": "flow"}} + assert result["params"]["today"] == "2026-10-21" async def test_the_segments_add_up_to_the_total_digit_for_digit(): @@ -163,7 +165,7 @@ async def test_drilling_down_checks_closure_against_the_parent_segment(): assert child["params"]["depth"] == 1 and child["params"]["parent"] == {"of": "A3", "key": ["华东"]} assert child["inputs"][0]["role"] == "parent" and child["inputs"][0]["ref"] == "A3" assert "max_drill_depth" in child["settings"] - assert {c["code"] for c in child["caveats"]} == set() # 有 parent 就不是「以各段之和为总数」 + assert {c["code"] for c in child["caveats"]} == {"comparability"} # 有 parent 就不是「以各段之和为总数」 short = snap("Q4", province["sql"], ["province", "week", "gmv"], [["沪", "2026-W41", 500], ["沪", "2026-W42", 410], ["苏", "2026-W41", 300], ["苏", "2026-W42", 290]]) unclosed = await analyze("decompose", "additive", snaps=[short], parents={"A3": top}, **params) @@ -504,3 +506,55 @@ async def test_a_dimension_named_like_a_result_column_is_a_call_error(): error = await call_error("decompose", "ratio", snaps=[mixed], query="Q1", by="Mix", period="week", base="W41", target="W42", numerator="refunds", denominator="orders") assert error.code == "reserved_column" and "「Mix」" in error.message + + + +# -------------------------------------------------------------------------- +# 一份快照按期次分组:本期齐不齐(同波动基线那一套) +# -------------------------------------------------------------------------- + +WEEK_START = ("SELECT region, date(order_date, '-6 days', 'weekday 1') AS week, SUM(amount) AS gmv FROM orders " + "GROUP BY 1, 2") +DAY_SQL = "SELECT region, order_date AS day, SUM(amount) AS gmv FROM orders GROUP BY 1, 2" + + +def _two(base, target, *, sql=WEEK_START, col="week"): + rows = [["华东", base, 820], ["华东", target, 700], ["华北", base, 600], ["华北", target, 560]] + return snap("Q2", sql, ["region", col, "gmv"], rows, types={"region": "text", col: "text", "gmv": "number"}) + + +async def test_one_snapshot_by_period_checks_the_target_is_whole(): + probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")}) + ask = {"query": "Q2", "by": ["region"], "period": "week", "measure": "gmv", "label": "销售额"} + # 运行当天 10-21:10-19 那一周没过完 + partial = await analyze("decompose", "additive", snaps=[_two("2026-10-12", "2026-10-19")], probe=probe, + base="2026-10-12", target="2026-10-19", **ask) + assert partial["reason_code"] == "partial_target" + assert partial["reason"] == "本期数据只到 2026-10-20,还差 5 天,和完整的周比没有意义" + assert partial["params"]["today"] == "2026-10-21" and partial["probes"][0]["column"] == "order_date" + whole = await analyze("decompose", "additive", snaps=[_two("2026-10-05", "2026-10-12")], probe=probe, + base="2026-10-05", target="2026-10-12", **ask) + assert whole["status"] == "ok" and not [c for c in whole["caveats"] if c["code"] == "comparability"] + # 查不到表的范围:只附一句核对不了,不进判定语 + unknown = await analyze("decompose", "additive", snaps=[_two("2026-10-05", "2026-10-12")], + base="2026-10-05", target="2026-10-12", **ask) + note = next(c["text"] for c in unknown["caveats"] if c["code"] == "comparability") + assert note == "没能核对本期的数据是否到齐(查不到表里这一列的日期范围)" and note not in unknown["phrase"] + ratio_rows = [["线上", "2026-10-12", 50, 1000], ["线上", "2026-10-19", 80, 1000], ["线下", "2026-10-12", 20, 1000], + ["线下", "2026-10-19", 26, 1000]] + ratio_snap = snap("Q1", "SELECT channel, date(order_date, '-6 days', 'weekday 1') AS week, SUM(refund_flag) AS " + "refunds, COUNT(*) AS orders FROM orders GROUP BY 1, 2", + ["channel", "week", "refunds", "orders"], ratio_rows) + refused = await analyze("decompose", "ratio", snaps=[ratio_snap], probe=probe, query="Q1", by="channel", + period="week", base="2026-10-12", target="2026-10-19", numerator="refunds", + denominator="orders") + assert refused["reason_code"] == "partial_target" + + +async def test_two_days_a_week_apart_from_a_plain_date_column_are_two_days(): + # 期次列直接取的日期:昨天(10-20)对上周同一天,不是两周;10-20 当天的数据已经到了 + probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")}) + result = await analyze("decompose", "additive", snaps=[_two("2026-10-13", "2026-10-20", sql=DAY_SQL, col="day")], + probe=probe, query="Q2", by=["region"], period="day", base="2026-10-13", + target="2026-10-20", measure="gmv") + assert result["status"] == "ok" and not [c for c in result["caveats"] if c["code"] == "comparability"] diff --git a/backend/tests/test_analysis_evidence_api.py b/backend/tests/test_analysis_evidence_api.py index 93b3b867..9c2c7568 100644 --- a/backend/tests/test_analysis_evidence_api.py +++ b/backend/tests/test_analysis_evidence_api.py @@ -136,7 +136,7 @@ async def test_the_phrase_opens_judgment_method_calculation_inputs_recompute_and assert judged["cited"] == {"what": "phrase"} and judged["render_ok"] is True and judged["eid_ok"] is True assert judged["origin"] == "card" and judged["item"] == "drop" and judged["label"] == "分析" assert judged["hypothesis"] == "华东拖累" and judged["hypothesis_note"].startswith("Agent 写的假设") - assert {c["code"] for c in judged["caveats"]} == {"measure_unlabeled", "headline_basis"} + assert {c["code"] for c in judged["caveats"]} == {"measure_unlabeled", "headline_basis", "comparability"} assert method["method"] == "decompose.additive@1" and method["method_label"] == "变化拆解(可加)" assert {s["key"]: s["label"] for s in method["settings"]}["main_share"] == "「主要来源」的占比" assert {s["key"] for s in method["settings"]} >= {"main_share", "concentration", "table_top_k"} diff --git a/backend/tests/test_analysis_periods.py b/backend/tests/test_analysis_periods.py index 241f3f1a..76ff612e 100644 --- a/backend/tests/test_analysis_periods.py +++ b/backend/tests/test_analysis_periods.py @@ -128,3 +128,15 @@ def test_unreadable_values_are_named(): assert pd.unreadable(["2026-W40", "上周", "2026-W41", None]) == ("shape", ["上周", None]) assert pd.unreadable(["2026-W40", "2026-W41", "2026-10-12"]) == ("mixed", ["2026-10-12"]) assert pd.unreadable(["2026-10", "2026-11"], "week") == ("grain", ["2026-10", "2026-11"]) + + +def test_a_plain_date_column_is_a_day_even_when_two_values_are_a_week_apart(): + plain = "SELECT region, order_date AS day, SUM(amount) AS gmv FROM orders GROUP BY 1, 2" + starts = ("SELECT region, date(order_date, '-6 days', 'weekday 1') AS week, SUM(amount) AS gmv FROM orders " + "GROUP BY 1, 2") + assert pd.plain_day(plain, "day", "sqlite") and pd.plain_day(plain.replace("order_date AS", "DATE(order_date) AS"), + "day", "sqlite") + assert not pd.plain_day(starts, "week", "sqlite") + assert pd.parse_compared(["2026-10-13", "2026-10-20"], plain, "day", "sqlite")[1] == "day" + assert pd.parse_compared(["2026-10-12", "2026-10-19"], starts, "week", "sqlite")[1] == "week" + assert pd.parse_compared(["2026-09-01", "2026-10-01"], plain, "day", "sqlite")[1] == "month" From 19bdcf55d87f0cfd241ce7d86a62e919fa1900f7 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 16:11:00 +0800 Subject: [PATCH 11/13] =?UTF-8?q?=E5=88=86=E6=9E=90=E5=8D=A1=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E6=97=B6=E5=AF=B9=E5=90=88=E5=B9=B6=E5=90=8E=E7=9A=84?= =?UTF-8?q?=E9=98=88=E5=80=BC=E5=86=8D=E6=A0=B8=E4=B8=80=E9=81=8D=EF=BC=88?= =?UTF-8?q?=E5=A4=8D=E6=A0=B8=20R28=EF=BC=89=EF=BC=9A=E7=94=BB=E5=9B=BE?= =?UTF-8?q?=E6=97=B6=E9=98=88=E5=80=BC=E8=A6=86=E7=9B=96=E5=8F=AA=E8=83=BD?= =?UTF-8?q?=E5=AF=B9=E7=9D=80=E7=BC=BA=E7=9C=81=E5=80=BC=E6=A0=A1=E9=AA=8C?= =?UTF-8?q?=EF=BC=88schema=20=E4=B8=8D=E6=94=B9=EF=BC=89=EF=BC=8C=E5=8F=91?= =?UTF-8?q?=E8=B5=B7=E6=97=B6=E7=9A=84=E8=AE=BE=E7=BD=AE=E5=BF=AB=E7=85=A7?= =?UTF-8?q?=E5=90=88=E8=BF=9B=E6=9D=A5=E4=BB=A5=E5=90=8E=E5=8F=AF=E8=83=BD?= =?UTF-8?q?=E5=86=B2=E7=AA=81=E2=80=94=E2=80=94=E7=AE=A1=E7=90=86=E5=91=98?= =?UTF-8?q?=E6=8A=8A=E3=80=8C=E8=B6=85=E5=87=BA=E5=B8=B8=E8=A7=84=E6=B3=A2?= =?UTF-8?q?=E5=8A=A8=E3=80=8D=E8=B0=83=E5=88=B0=203=E3=80=81=E5=8D=A1?= =?UTF-8?q?=E4=B8=8A=E6=8A=8A=E3=80=8C=E6=98=8E=E6=98=BE=E8=B6=85=E5=87=BA?= =?UTF-8?q?=E3=80=8D=E8=A6=86=E7=9B=96=E6=88=90=202.5=EF=BC=8C=E5=B0=B1?= =?UTF-8?q?=E6=B0=B8=E8=BF=9C=E5=88=A4=E4=B8=8D=E5=87=BA=E8=B6=85=E5=87=BA?= =?UTF-8?q?=E3=80=82=E5=8D=A1=E4=B8=8A=E5=86=99=E4=BA=86=E8=A6=86=E7=9B=96?= =?UTF-8?q?=E6=97=B6=EF=BC=8C=E8=BF=90=E8=A1=8C=E6=97=B6=E5=AF=B9=E7=BC=BA?= =?UTF-8?q?=E7=9C=81=20=E2=86=90=20=E5=BF=AB=E7=85=A7=20=E2=86=90=20?= =?UTF-8?q?=E8=A6=86=E7=9B=96=E5=90=88=E5=B9=B6=E5=90=8E=E7=9A=84=E9=98=88?= =?UTF-8?q?=E5=80=BC=E8=B7=91=E4=B8=80=E6=AC=A1=20contract.settings=5Fprob?= =?UTF-8?q?lem=EF=BC=8C=E6=9C=89=E9=97=AE=E9=A2=98=E8=AE=A9=E5=8D=A1?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E5=B9=B6=E8=AF=B4=E6=B8=85=E3=80=8C=E9=98=88?= =?UTF-8?q?=E5=80=BC=E8=A6=86=E7=9B=96=E5=92=8C=E8=BF=99=E6=AC=A1=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E7=9A=84=E5=88=86=E6=9E=90=E8=AE=BE=E7=BD=AE=E5=86=B2?= =?UTF-8?q?=E7=AA=81=EF=BC=9A=E2=80=A6=E3=80=82=E8=AF=B7=E6=94=B9=E5=88=86?= =?UTF-8?q?=E6=9E=90=E5=8D=A1=E4=B8=8A=E7=9A=84=E9=98=88=E5=80=BC=E8=A6=86?= =?UTF-8?q?=E7=9B=96=E3=80=8D=EF=BC=88=E9=85=8D=E7=BD=AE=E9=94=99=E8=AF=AF?= =?UTF-8?q?=EF=BC=8C=E4=B8=8D=E6=98=AF=E6=95=B0=E6=8D=AE=E9=97=AE=E9=A2=98?= =?UTF-8?q?=EF=BC=8C=E4=B8=80=E9=A1=B9=E9=83=BD=E4=B8=8D=E7=AE=97=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/engine/nodes/analysis.py | 7 ++++++- backend/tests/test_analysis_card_data.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/backend/app/engine/nodes/analysis.py b/backend/app/engine/nodes/analysis.py index 80501458..3c622232 100644 --- a/backend/app/engine/nodes/analysis.py +++ b/backend/app/engine/nodes/analysis.py @@ -56,7 +56,12 @@ async def run_analysis(state: GraphState, ctx: NodeContext) -> dict[str, Any]: if errors := [i for i in analysis_issues(ctx.run.spec, ctx.node) if i.level == "error"]: raise NodeError(ctx.node.id, errors[0].message) items: list[dict[str, Any]] = ctx.config["items"] - settings = await settings_of(run_snapshot(ctx.run), ctx.config.get("overrides") or None) + overrides = ctx.config.get("overrides") or None + settings = await settings_of(run_snapshot(ctx.run), overrides) + # 画图时阈值覆盖只能对着缺省值校验;合并了这次运行的设置快照以后再核一遍(管理员把「超出常规波动」调到 3、卡上把「明显 + # 超出」覆盖成 2.5,就永远判不出超出)。这是配置错误,不是数据问题:让卡失败、说清冲突在哪 + if overrides and (problem := contract.settings_problem(settings)): + raise NodeError(ctx.node.id, f"阈值覆盖和这次运行的分析设置冲突:{problem}。请改分析卡上的阈值覆盖") queries = [e for e in state.get("evidence") or [] if isinstance(e, dict) and e.get("kind") == "query"] inputs: dict[str, InputSnapshot] = {} diff --git a/backend/tests/test_analysis_card_data.py b/backend/tests/test_analysis_card_data.py index 755df707..179ef740 100644 --- a/backend/tests/test_analysis_card_data.py +++ b/backend/tests/test_analysis_card_data.py @@ -68,3 +68,16 @@ async def test_a_structural_call_error_still_fails_the_card_and_names_the_item() assert row.status == "failed" assert "分析项「where」(变化拆解)" in row.error and "没有「channel」这一列" in row.error assert "请在分析卡中改正这一项" in row.error + + +async def test_overrides_that_conflict_with_the_run_settings_fail_the_card(): + # 画图时对着缺省值校验能过(缺省「超出常规波动」是 2);这次运行的设置把它调到 3,卡上覆盖的「明显超出」2.5 就永远判不出 + card = node("card", "analysis", items=[WHERE], overrides={"k_alert": 2.5}) + graph = chain(node("start", "input"), node("by_region", "tool", tool=TOOL, args={"sql": BY_REGION}), card) + run = await run_manager.start(graph=graph, input_payload={}, options={"analysis": {"k_warn": 3.0}}) + row = await wait(run.id) + assert row.status == "failed" + assert "阈值覆盖和这次运行的分析设置冲突:「明显超出」的倍数要大于「超出常规波动」的倍数" in row.error + assert await events(row.id, "analysis.end", "card") == [] + ok = await run_manager.start(graph=graph, input_payload={}, options={"analysis": {"k_warn": 2.0}}) + assert (await wait(ok.id)).status == "succeeded" From f5e6e77ea9fd06ba2f3ed2c21a5fbde730c2dccf Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 16:16:40 +0800 Subject: [PATCH 12/13] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=94=A8=E7=9A=84?= =?UTF-8?q?=E5=81=87=E7=BB=93=E8=AE=BA=E6=8C=89=E5=90=84=E7=AE=97=E5=AD=90?= =?UTF-8?q?=E7=9A=84=E5=BD=A2=E7=8A=B6=E7=BB=99=EF=BC=88=E5=A4=8D=E6=A0=B8?= =?UTF-8?q?=20R31=EF=BC=89=EF=BC=9Atesting.fake=5Fresult=20=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E4=B8=80=E5=BE=8B=E6=8B=BF=E5=8F=AF=E5=8A=A0=E6=8B=86?= =?UTF-8?q?=E8=A7=A3=E7=9A=84=E7=BB=93=E6=9E=9C=E8=A1=A8=E2=80=94=E2=80=94?= =?UTF-8?q?=E6=B3=A2=E5=8A=A8=E5=9F=BA=E7=BA=BF=EF=BC=88=E6=9C=9F=E6=AC=A1?= =?UTF-8?q?=E3=80=81value=E3=80=81change=E3=80=81role=EF=BC=89=E3=80=81?= =?UTF-8?q?=E8=B6=8B=E5=8A=BF=EF=BC=88=E6=9C=9F=E6=AC=A1=E3=80=81value?= =?UTF-8?q?=E3=80=81change=EF=BC=89=E3=80=81=E5=B7=AE=E5=BC=82=E6=A3=80?= =?UTF-8?q?=E9=AA=8C=EF=BC=88a=E3=80=81b=E3=80=81a=5Fvalue=E2=80=A6?= =?UTF-8?q?=E2=80=A6p=5Fadj=E3=80=81flag=EF=BC=89=E3=80=81=E5=8F=AF?= =?UTF-8?q?=E6=AF=94=E6=80=A7=E6=A3=80=E6=9F=A5=EF=BC=88check=E3=80=81stat?= =?UTF-8?q?us=E3=80=81base=E3=80=81target=E3=80=81note=EF=BC=89=E5=90=84?= =?UTF-8?q?=E7=BB=99=E8=87=AA=E5=B7=B1=E5=BD=A2=E7=8A=B6=E7=9A=84=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E8=A1=A8=EF=BC=88=E5=B8=A6=E5=88=97=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=EF=BC=89=E3=80=81=E5=85=B7=E5=90=8D=E6=A0=87=E9=87=8F=E3=80=81?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E5=92=8C=E8=BE=93=E5=85=A5=EF=BC=88=E5=8F=AF?= =?UTF-8?q?=E6=AF=94=E6=80=A7=E6=A3=80=E6=9F=A5=E6=98=AF=E5=9F=BA=E6=9C=9F?= =?UTF-8?q?=E3=80=81=E6=9C=AC=E6=9C=9F=E4=B8=A4=E4=BB=BD=EF=BC=89=EF=BC=8C?= =?UTF-8?q?=E6=95=B0=E6=98=AF=E7=BC=96=E7=9A=84=E3=80=81=E5=92=8C=E5=88=A4?= =?UTF-8?q?=E5=AE=9A=E8=AF=AD=E5=AF=B9=E5=BE=97=E4=B8=8A=EF=BC=9B=E5=B7=AE?= =?UTF-8?q?=E5=BC=82=E6=A3=80=E9=AA=8C=E6=B2=A1=E5=86=99=E6=A8=A1=E5=BC=8F?= =?UTF-8?q?=E6=97=B6=E6=8C=89=E6=AF=94=E7=8E=87=E3=80=82=E5=8F=98=E5=8C=96?= =?UTF-8?q?=E6=8B=86=E8=A7=A3=E9=82=A3=E4=B8=80=E4=BB=BD=E7=85=A7=E6=97=A7?= =?UTF-8?q?=E3=80=82=E6=B5=8B=E8=AF=95=E5=AF=B9=E7=9D=80=E7=9C=9F=E7=AE=97?= =?UTF-8?q?=E5=AD=90=E6=A0=B8=E5=AF=B9=E5=88=97=E5=92=8C=E5=85=B7=E5=90=8D?= =?UTF-8?q?=E6=A0=87=E9=87=8F=E7=9A=84=E9=94=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/testing.py | 121 +++++++++++++++++++++++++---- backend/tests/test_analysis_run.py | 12 +++ 2 files changed, 117 insertions(+), 16 deletions(-) diff --git a/backend/app/analysis/testing.py b/backend/app/analysis/testing.py index 1dbbc43d..538c3792 100644 --- a/backend/app/analysis/testing.py +++ b/backend/app/analysis/testing.py @@ -1,5 +1,8 @@ """测试用的分析结论(只给测试用):第 1 波各包并行开发时,证据接入、入口、说法核对的测试不等算子写完,按契约的形状 -拿一份假的结论。算子落地后,端到端的测试改用 analysis.run 真算。""" +拿一份假的结论。算子落地后,端到端的测试改用 analysis.run 真算。 + +每个算子的假结论带它自己形状的结果表、具名标量、参数和输入(列 id、叫法、类型照各算子的实现:变化拆解按维度、波动基线和 +趋势按期次、差异检验按对、可比性检查按检查项),数是编的、彼此对得上判定语。""" from __future__ import annotations from typing import Any @@ -18,14 +21,11 @@ _PHRASES = { "incomparable": "两期不可比:本期只有 3 天数据", } +_WEEKS = ["2026-07-20", "2026-07-27", "2026-08-03", "2026-08-10", "2026-08-17", "2026-08-24", "2026-08-31", + "2026-09-07", "2026-09-14", "2026-09-21", "2026-09-28", "2026-10-05", "2026-10-12"] -def fake_result(op: str = "decompose", mode: str | None = "additive", verdict: str | None = "driver", *, - subjects: list[list[str]] | None = None, direction: str | None = "down", status: str = "ok", - reason_code: str | None = None, inputs: list[dict[str, Any]] | None = None, - **overrides: Any) -> AnalysisResult: - """一份形状完整的假结论。缺省是「销售额下降主要来自华东」的可加拆解。""" - if op not in ("decompose", "differ"): - mode = None + +def _decompose() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]: table = { "columns": ["region", "base", "target", "delta", "pct", "share", "flag"], "labels": {"region": "地区", "base": "基期", "target": "本期", "delta": "变化", "pct": "变化率", @@ -35,26 +35,115 @@ def fake_result(op: str = "decompose", mode: str | None = "additive", verdict: s ["华南", "700", "666", "-34", "-0.0486", "0.1753", None], ["西南", "100", "100", "0", "0", "0", None]], } + values = {"base_total": "2220", "target_total": "2026", "delta_total": "-194", "pct_total": "-0.0874", + "top_key": "华东", "top_share": "0.6186", "segments": "4"} + params = {"query": "Q2", "by": ["region"], "period": "week", "base": "W41", "target": "W42", "measure": "gmv", + "label": "销售额"} + return table, values, params, ["region", "week", "gmv"] + + +def _baseline() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]: + changes = ["0.012", "-0.021", "0.034", "-0.008", "0.019", "-0.045", "0.027", "-0.015", "0.006", "0.041", "-0.032", + "-0.004"] + levels = ["2210", "2164", "2238", "2220", "2262", "2160", "2218", "2185", "2198", "2288", "2215", "2206"] + columns = ["week", "value", "change", "role"] + rows = [[w, v, c, "历史"] for w, v, c in zip(_WEEKS[:-1], levels, changes)] + [[_WEEKS[-1], "2138", "-0.031", "本期"]] + table = {"columns": columns, "labels": {"week": "周", "value": "销售额", "change": "变化率", "role": "说明"}, + "rows": rows, "column_types": {"week": "text", "value": "number", "change": "number", "role": "text"}} + values = {"x": "-0.031", "value": "2138", "median": "0.001", "band_lo": "-0.061", "band_hi": "0.053", "z": "-1.12", + "rank": "11", "n_history": "12", "hist_min": "-0.045", "hist_max": "0.041"} + params = {"query": "Q1", "period": "week", "measure": "gmv", "label": "销售额", "grain": "week", "basis": "change", + "lag": 1, "target": _WEEKS[-1], "today": "2026-10-21"} + return table, values, params, ["week", "gmv"] + + +def _trend() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]: + levels = [100, 103, 106, 109, 113, 116, 119, 123, 127, 130] + rows = [[w, str(v), str(v - levels[i - 1]) if i else None] for i, (w, v) in enumerate(zip(_WEEKS[-10:], levels))] + table = {"columns": ["week", "value", "change"], "labels": {"week": "周", "value": "线上销售额", "change": "比上一期"}, + "rows": rows, "column_types": {"week": "text", "value": "number", "change": "number"}} + values = {"slope": "3.333333333333", "rel_slope": "0.03", "p": "0.000002755732", "s": "45", "n": "10", + "run_length": "9", "run_direction": "up"} + params = {"query": "Q1", "period": "week", "measure": "gmv", "label": "线上销售额", "grain": "week", + "today": "2026-10-21"} + return table, values, params, ["week", "gmv"] + + +def _differ() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]: + columns = ["a", "b", "a_value", "b_value", "diff", "ci_lo", "ci_hi", "a_n", "b_n", "test", "p", "p_adj", "flag"] + table = {"columns": columns, + "labels": {"a": "组", "b": "对照组", "a_value": "组的比率", "b_value": "对照组的比率", "diff": "差", + "ci_lo": "差的区间下限", "ci_hi": "差的区间上限", "a_n": "组的样本量", "b_n": "对照组的样本量", + "test": "检验", "p": "p 值", "p_adj": "校正后的 p 值", "flag": "说明"}, + "rows": [["西南", "华东", "0.05", "0.021739130435", "0.028260869565", "-0.026", "0.137", "40", "230", + "Fisher 精确检验", "0.27", "0.27", "样本少,用了精确检验"]], + "column_types": {"a": "text", "b": "text", **{c: "number" for c in columns[2:9]}, "test": "text", + "p": "number", "p_adj": "number", "flag": "text"}} + values = {"a": "0.05", "b": "0.021739130435", "diff": "0.028260869565", "ci_lo": "-0.026", "ci_hi": "0.137", + "p": "0.27", "p_adj": "0.27", "n_a": "40", "n_b": "230", "test": "fisher", "pairs": "1", "differs": "0"} + params = {"query": "Q3", "by": ["region"], "numerator": "refunds", "denominator": "orders", "label": "退款率", + "pairs": [["西南", "华东"]]} + return table, values, params, ["region", "refunds", "orders"] + + +def _comparable() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]: + columns = ["check", "status", "base", "target", "note"] + table = {"columns": columns, + "labels": {"check": "检查项", "status": "结果", "base": "基期", "target": "本期", "note": "说明"}, + "rows": [["同口径", "通过", None, None, None], + ["窗口长度", "通过", "2026-10-05 到 2026-10-11(7 天)", "2026-10-12 到 2026-10-18(7 天)", None], + ["整期", "通过", "通过", "通过", None], ["星期构成", "通过", None, None, None], + ["数据问题", "通过", None, None, None]], + "column_types": dict.fromkeys(columns, "text")} + values = {"same_query": "pass", "window": "pass", "complete": "pass", "weekdays": "pass", "data": "pass", + "base_start": "2026-10-05", "base_end": "2026-10-11", "base_days": "7", "base_weekend": "2", + "target_start": "2026-10-12", "target_end": "2026-10-18", "target_days": "7", "target_weekend": "2"} + params = {"base_query": "Q4", "target_query": "Q5", "measure": "gmv", "today": "2026-10-21"} + return table, values, params, ["gmv"] + + +_SHAPES = {"decompose": _decompose, "baseline": _baseline, "trend": _trend, "differ": _differ, + "comparable": _comparable} +_CRITERIA = {"decompose": "一段占总变化一半以上算主要来源", + "baseline": "本期和过去 12 周的中位数比,差不到 2 倍稳健标准差算常规波动", + "trend": "Mann–Kendall 检验的双侧 p 值小于 0.05 时按 Theil–Sen 斜率的正负判上升或下降", + "differ": "双侧 p 值小于 0.05 算差异站得住", + "comparable": "只差日期、窗口一样长、是整期、星期构成相同才算可比"} + + +def fake_result(op: str = "decompose", mode: str | None = "additive", verdict: str | None = "driver", *, + subjects: list[list[str]] | None = None, direction: str | None = "down", status: str = "ok", + reason_code: str | None = None, inputs: list[dict[str, Any]] | None = None, + **overrides: Any) -> AnalysisResult: + """一份形状完整的假结论。缺省是「销售额下降主要来自华东」的可加拆解;别的算子给各自形状的结果表、具名标量和参数。""" + if op not in ("decompose", "differ"): + mode = None + elif op == "differ" and mode not in ("rate", "mean", "rows"): + mode = "rate" + table, values, params, columns = _SHAPES.get(op, _decompose)() + if op == "comparable": + default_inputs = [{"ref": ref, "artifact": "0" * 64, "role": role, "columns": columns, "rows": "all"} + for ref, role in (("Q4", "base"), ("Q5", "target"))] + else: + default_inputs = [{"ref": str(params.get("query") or "Q2"), "artifact": "0" * 64, "role": "data", + "columns": columns, "rows": "all"}] result: AnalysisResult = { "op": op, "mode": mode, "method": METHODS.get((op, mode), f"{op}@1"), - "params": {"query": "Q2", "by": ["region"], "period": "week", "base": "W41", "target": "W42", - "measure": "gmv", "label": "销售额"}, + "params": params, "settings": effective_settings(None), - "inputs": inputs if inputs is not None else [ - {"ref": "Q2", "artifact": "0" * 64, "role": "data", "columns": ["region", "week", "gmv"], "rows": "all"}], + "inputs": inputs if inputs is not None else default_inputs, "probes": [], "status": status, # type: ignore[typeddict-item] "reason_code": reason_code, "reason": None if status == "ok" else "历史只有 5 期,至少要 8 期", - "table": table if status == "ok" else None, - "values": {"base_total": "2220", "target_total": "2026", "delta_total": "-194", "pct_total": "-0.0874", - "top_key": "华东", "top_share": "0.6186", "segments": "4"} if status == "ok" else {}, + "table": table if status == "ok" else None, # type: ignore[typeddict-item] + "values": values if status == "ok" else {}, "verdict": ({"code": verdict, "subjects": subjects if subjects is not None else ([["华东"]] if verdict == "driver" else []), "direction": direction} # type: ignore[typeddict-item] if status == "ok" and verdict else None), - "criterion": "一段占总变化一半以上算主要来源", + "criterion": _CRITERIA.get(op, _CRITERIA["decompose"]), "phrase": _PHRASES.get(verdict or "", "") if status == "ok" else "变化拆解没有做成:历史只有 5 期,至少要 8 期", "caveats": [], "hypothesis": "地区拖累", diff --git a/backend/tests/test_analysis_run.py b/backend/tests/test_analysis_run.py index 36c09a44..85b07b9a 100644 --- a/backend/tests/test_analysis_run.py +++ b/backend/tests/test_analysis_run.py @@ -247,3 +247,15 @@ async def test_structural_call_errors_still_fail_even_when_data_errors_are_refus with pytest.raises(CallError) as e: await analyze("decompose", "additive", snaps=REFUSALS, refuse_data_errors=True, check=False, **params) assert e.value.code not in analysis.DATA_ERRORS and e.value.reason is None + + +@pytest.mark.parametrize("op,mode,params", [CALLS[0], *CALLS[2:]]) +async def test_the_fake_results_for_tests_have_each_operators_shape(op, mode, params): + from app.analysis.testing import fake_result + + real = await _run(op, mode, params) + fake = fake_result(op, mode, real["verdict"]["code"]) + assert fake["method"] == real["method"] and fake["table"]["columns"] == real["table"]["columns"] + assert set(fake["table"]["labels"]) == set(fake["table"]["columns"]) + assert set(fake["values"]) <= set(real["values"]) and {i["role"] for i in fake["inputs"]} == \ + {i["role"] for i in real["inputs"]} From 2efea2b753c7056e438d237bee58de0c96b282cc Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 16:56:48 +0800 Subject: [PATCH 13/13] =?UTF-8?q?=E5=91=A8=E5=8F=B7=E8=B6=85=E5=87=BA?= =?UTF-8?q?=E4=BB=BB=E4=BD=95=E7=AE=97=E6=B3=95=EF=BC=88W54=20=E4=BB=A5?= =?UTF-8?q?=E4=B8=8A=EF=BC=89=E7=9A=84=E8=AE=A4=E4=B8=8D=E5=87=BA=E3=80=81?= =?UTF-8?q?=E7=AE=97=E8=B5=B7=E6=AD=A2=E8=B6=85=E5=87=BA=E6=97=A5=E6=9C=9F?= =?UTF-8?q?=E8=8C=83=E5=9B=B4=E7=9A=84=E5=BD=93=E8=BF=99=E4=B8=80=E5=91=A8?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=EF=BC=88=E5=A4=8D=E6=A0=B8=20R3=20=E8=A1=A5?= =?UTF-8?q?=EF=BC=89=EF=BC=9A=E8=AE=A4=E4=B8=8D=E5=87=BA=E7=AE=97=E6=B3=95?= =?UTF-8?q?=E6=97=B6=E5=8E=9F=E6=9D=A5=E6=8C=89=E5=91=A8=E5=8F=B7=E6=8E=92?= =?UTF-8?q?=E5=85=88=E5=90=8E=EF=BC=8C=E3=80=8C2026-W99=E3=80=8D=E4=B9=9F?= =?UTF-8?q?=E4=BC=9A=E5=BD=93=E4=B8=80=E5=91=A8=E6=94=B6=E4=B8=8B=EF=BC=8C?= =?UTF-8?q?=E7=8E=B0=E5=9C=A8=E5=91=A8=E5=8F=B7=E5=A4=A7=E4=BA=8E=2053=20?= =?UTF-8?q?=E7=9A=84=E4=B8=8D=E5=BD=93=E5=91=A8=E5=8F=B7=E5=86=99=E6=B3=95?= =?UTF-8?q?=EF=BC=8C=E5=8E=9F=E5=9B=A0=E9=87=8C=E7=85=A7=E5=AE=9E=E4=B8=BE?= =?UTF-8?q?=E5=AE=83=EF=BC=9B=E6=8C=89=E7=AE=97=E6=B3=95=E7=AE=97=E8=B5=B7?= =?UTF-8?q?=E6=AD=A2=E6=97=B6=E6=97=A5=E5=AD=90=E8=B6=85=E5=87=BA=E6=97=A5?= =?UTF-8?q?=E6=9C=9F=E8=83=BD=E8=A1=A8=E7=A4=BA=E7=9A=84=E8=8C=83=E5=9B=B4?= =?UTF-8?q?=EF=BC=88=E5=85=AC=E5=85=83=209999=20=E5=B9=B4=E6=9C=AB?= =?UTF-8?q?=EF=BC=89=E4=BA=A4=E5=9B=9E=E6=B2=A1=E6=9C=89=E8=BF=99=E4=B8=80?= =?UTF-8?q?=E5=91=A8=EF=BC=8C=E4=B8=8D=E6=8A=9B=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/periods.py | 21 +++++++++++---------- backend/tests/test_analysis_periods.py | 7 +++++++ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/backend/app/analysis/periods.py b/backend/app/analysis/periods.py index 63e6fd28..fd29fab1 100644 --- a/backend/app/analysis/periods.py +++ b/backend/app/analysis/periods.py @@ -126,7 +126,7 @@ def _shape(value: Any) -> str | None: text = value.strip() if (m := _MONTH.match(text)) and 1 <= int(m.group(2)) <= 12 and 1900 <= int(m.group(1)) <= 2200: return "month" - if _WEEK.match(text): + if (m := _WEEK.match(text)) and int(m.group(2)) <= 53: return "week" if _QUARTER.match(text): return "quarter" @@ -161,17 +161,18 @@ def _first_weekday(start: date, weekday: int) -> date: def week_span(scheme: str, year: int, week: int) -> tuple[date, date] | None: """按周号算法,year 年第 week 周覆盖的第一天和最后一天(含)。这种算法里没有这一周(ISO 只有 52 周的年份写了第 53 周、 - 1 月 1 日就是周一的年份写了第 0 周……)为 None。""" - if scheme == "iso": - try: - start = date.fromisocalendar(year, week, 1) - except ValueError: - return None - return start, start + timedelta(days=6) + 1 月 1 日就是周一的年份写了第 0 周……)、日子超出日期能表示的范围的为 None。""" try: - jan1, dec31 = date(year, 1, 1), date(year, 12, 31) - except ValueError: + return _week_span(scheme, year, week) + except (OverflowError, ValueError): return None + + +def _week_span(scheme: str, year: int, week: int) -> tuple[date, date] | None: + if scheme == "iso": + start = date.fromisocalendar(year, week, 1) + return start, start + timedelta(days=6) + jan1, dec31 = date(year, 1, 1), date(year, 12, 31) if scheme == "jan1": if not 1 <= week <= 53: return None diff --git a/backend/tests/test_analysis_periods.py b/backend/tests/test_analysis_periods.py index 76ff612e..07462b29 100644 --- a/backend/tests/test_analysis_periods.py +++ b/backend/tests/test_analysis_periods.py @@ -140,3 +140,10 @@ def test_a_plain_date_column_is_a_day_even_when_two_values_are_a_week_apart(): assert pd.parse_compared(["2026-10-13", "2026-10-20"], plain, "day", "sqlite")[1] == "day" assert pd.parse_compared(["2026-10-12", "2026-10-19"], starts, "week", "sqlite")[1] == "week" assert pd.parse_compared(["2026-09-01", "2026-10-01"], plain, "day", "sqlite")[1] == "month" + + + +def test_week_numbers_out_of_any_algorithm_are_unreadable(): + assert pd.parse(["2026-W41", "2026-W99"]) is None + assert pd.unreadable(["2026-W41", "2026-W99"]) == ("shape", ["2026-W99"]) + assert pd.week_span("sunday_year", 9999, 53) is None and pd.week_span("iso", 0, 1) is None