mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 05:31:59 +00:00
合入复核修复(算子与分析卡运行器):周号按 SQL 的格式串算真实起止、分析卡里随数据变化的错误记成算不了、同口径先去排序再比、拿不到 p 值判算不了、敏感性只在真实各组找、叫法消毒幂等且不许夹带判断词、趋势连续期数看方向、闭合容差、集中不小于主要、重名列报错、单份快照拆解核本期齐不齐、阈值覆盖运行时再核
This commit is contained in:
@@ -5,8 +5,11 @@
|
||||
- run:按请求算一份分析结论。resolve 按调用里写的名字(Q3、来源节点 id)取回输入快照;单元格写成「名字.r行.列」
|
||||
(Q1.r0.gmv),由这里拆开再经 resolve 取快照。parents 按名字(A3、分析卡里的项 id)取回上一层拆解,下钻用。
|
||||
调用写错抛 contract.CallError;算不了返回 status refused 的结论。唯一的异步来自 context["probe"](现查表的日期范围)。
|
||||
refuse_data_errors 为真时(分析卡:调用是模板定死的,这期的数据里没有要用的那一项不是写错了),随数据变化的调用错误
|
||||
(DATA_ERRORS)也记成算不了的结论:上一层没做成 parent_refused,其余 input_unavailable,原因照实写是哪一项、哪个值;
|
||||
结构性的(列不存在、遮罩列、比率当可加、重复行、参数写错)照旧抛。agent 工具不传,调用错误交回模型改调用。
|
||||
- recompute:按结论里记下的 params、settings、probes 和输入快照(按 sha256 取回)重算一遍,不现查;和封存的用
|
||||
contract.same 比。
|
||||
contract.same 比。一律按 refuse_data_errors 为真复现:分析卡记下的这类结论复算也成立。
|
||||
|
||||
契约(名字、形状、缺省阈值)在 contract.py。各算子在 decompose.py(变化拆解)、series.py(波动基线、趋势)、differ.py
|
||||
(差异检验)、comparable.py(可比性检查);统计在 stats.py,判定语和数的写法在 phrases.py,现查在 probe.py。
|
||||
@@ -32,6 +35,14 @@ Resolver = Callable[[str], InputSnapshot | None]
|
||||
#: 按名字取回上一层的分析结论(下钻);没有返回 None
|
||||
ParentResolver = Callable[[str], AnalysisResult | None]
|
||||
|
||||
#: 随数据变化的调用错误(CallError.code):调用本身没写错,是这期的数据里没有要用的那一项。分析卡里(refuse_data_errors)
|
||||
#: 记成算不了的结论,不让卡失败:上一层没做成(parent_refused);下钻的段这期没有(parent_key_missing);期次这期没有
|
||||
#: (period_not_found:变化拆解、波动基线的本期、差异检验按期比、可比性检查);按位置取的总数那一格越界或为空
|
||||
#: (row_out_of_range、headline_empty);要比的组这期没有(unknown_group、too_few_groups);要用的那一格这期为空
|
||||
#: (empty_cell:乘积拆解的乘积、因子,差异检验的计数)
|
||||
DATA_ERRORS = frozenset({"parent_refused", "parent_key_missing", "period_not_found", "row_out_of_range",
|
||||
"headline_empty", "unknown_group", "too_few_groups", "empty_cell"})
|
||||
|
||||
|
||||
def _op_mode(request: Mapping[str, Any]) -> tuple[str, str | None]:
|
||||
op = request.get("op")
|
||||
@@ -69,9 +80,44 @@ def _env(op: str, mode: str | None, settings: Mapping[str, Any], inputs: Any, fa
|
||||
parents=parents, hypothesis=hypothesis if isinstance(hypothesis, str) else None)
|
||||
|
||||
|
||||
def _data_refusal(op: str, mode: str | None, raw: Mapping[str, Any], env: Any, error: CallError) -> AnalysisResult:
|
||||
"""随数据变化的调用错误记成的算不了的结论。params 记调用原样(label 照样消毒;调用还没走完、规范化不完整,复算拿它重走一遍
|
||||
走到同一处),取过的快照都记进输入(Inputs.settle),阈值和判定标准同这个算子自己算不了时那一份。"""
|
||||
from app.analysis import decompose, differ, series
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis.common import build
|
||||
|
||||
env.inputs.settle()
|
||||
params = dict(raw)
|
||||
if params.get("label") is not None:
|
||||
params["label"] = ph.clean_label(params["label"]) or None
|
||||
if op == "decompose":
|
||||
used, criterion = decompose.refusal_basis(mode, raw, env.settings) # type: ignore[arg-type]
|
||||
elif op in ("baseline", "trend"):
|
||||
used, criterion = series.refusal_basis(op, env.settings)
|
||||
elif op == "differ":
|
||||
used, criterion = differ.refusal_basis(mode, env.settings)
|
||||
else:
|
||||
used, criterion = [], ph.CRITERION_COMPARABLE
|
||||
code = "parent_refused" if error.code == "parent_refused" else "input_unavailable"
|
||||
reason = error.reason or error.message
|
||||
return build(env, params, used, status="refused", reason_code=code, reason=reason, criterion=criterion,
|
||||
phrase=ph.refused(op, reason))
|
||||
|
||||
|
||||
def _compute_or_refuse(op: str, mode: str | None, raw: Mapping[str, Any], env: Any, refuse_data_errors: bool
|
||||
) -> AnalysisResult:
|
||||
try:
|
||||
return _compute(op, mode, raw, env)
|
||||
except CallError as e:
|
||||
if refuse_data_errors and e.code in DATA_ERRORS:
|
||||
return _data_refusal(op, mode, raw, env, e)
|
||||
raise
|
||||
|
||||
|
||||
async def run(request: AnalysisRequest, resolve: Resolver, *, settings: Mapping[str, Any], context: Context,
|
||||
parents: ParentResolver | None = None) -> AnalysisResult:
|
||||
"""算一份分析结论。settings 是 contract.effective_settings 算好的阈值。"""
|
||||
parents: ParentResolver | None = None, refuse_data_errors: bool = False) -> AnalysisResult:
|
||||
"""算一份分析结论。settings 是 contract.effective_settings 算好的阈值。refuse_data_errors 见文件头(分析卡传真)。"""
|
||||
from app.analysis.common import DERIVED_PARAMS
|
||||
from app.analysis.facts import Facts
|
||||
from app.analysis.inputs import Inputs
|
||||
@@ -97,7 +143,7 @@ async def run(request: AnalysisRequest, resolve: Resolver, *, settings: Mapping[
|
||||
book = Book(known)
|
||||
env = _env(op, mode, full, Inputs(resolve, dialect), Facts(catalogs=context.get("catalogs")), book, today,
|
||||
parents, request.get("hypothesis"))
|
||||
result = _compute(op, mode, raw, env)
|
||||
result = _compute_or_refuse(op, mode, raw, env, refuse_data_errors)
|
||||
missing = book.missing
|
||||
if not missing or rounds == MAX_ROUNDS:
|
||||
return result
|
||||
@@ -109,7 +155,8 @@ def recompute(result: AnalysisResult, load: Callable[[str], InputSnapshot | None
|
||||
load_result: Callable[[str], AnalysisResult | None] | None = None) -> AnalysisResult:
|
||||
"""复算:load 按 sha256 取回输入快照,load_result 按 sha256 取回 parent 那份结论。用结论里记下的 params(含运行当天、
|
||||
目录事实)、settings、probes,不现查、不读目录。取不回输入时抛 CallError(code input_missing);方法升版后调用不再成立
|
||||
时同样抛 CallError(照 run 的说法)。和封存的那份用 contract.same 比。"""
|
||||
时同样抛 CallError(照 run 的说法)。随数据变化的调用错误一律记成算不了(同 run 的 refuse_data_errors 为真),分析卡记下
|
||||
的这类结论复算也成立。和封存的那份用 contract.same 比。"""
|
||||
from app.analysis.common import DERIVED_PARAMS
|
||||
from app.analysis.facts import Facts
|
||||
from app.analysis.inputs import _CELL, Inputs
|
||||
@@ -150,7 +197,7 @@ def recompute(result: AnalysisResult, load: Callable[[str], InputSnapshot | None
|
||||
env = _env(op, mode, settings, Inputs(resolve, dialect), Facts(recorded=params.get("catalog") or {}),
|
||||
Book(result.get("probes") or []), today, parents, result.get("hypothesis"))
|
||||
raw = {k: v for k, v in params.items() if k not in DERIVED_PARAMS}
|
||||
return _compute(op, mode, raw, env)
|
||||
return _compute_or_refuse(op, mode, raw, env, True)
|
||||
|
||||
|
||||
def exhausted(settings: Mapping[str, Any]) -> CallError:
|
||||
@@ -174,4 +221,5 @@ def comparable_reasons(result: Mapping[str, Any]) -> list[str]:
|
||||
return reasons(result)
|
||||
|
||||
|
||||
__all__ = ["ParentResolver", "Resolver", "comparable_reasons", "exhausted", "make_prober", "recompute", "run"]
|
||||
__all__ = ["DATA_ERRORS", "ParentResolver", "Resolver", "comparable_reasons", "exhausted", "make_prober", "recompute",
|
||||
"run"]
|
||||
|
||||
@@ -106,23 +106,35 @@ def need(raw: Mapping[str, Any], key: str, what: str) -> Any:
|
||||
return value
|
||||
|
||||
|
||||
def reserved_check(column: str, reserved: Sequence[str], what: str) -> None:
|
||||
"""结果表的列 id 是输入的维度列(期次列)加上固定的几列:重名了列 id 就重复,引用结果表的格、下钻按的维度数都会认错。
|
||||
遇到这些名字(不分大小写)是调用错误,让 agent 在 SQL 里换个别名。"""
|
||||
if column.lower() in {r.lower() for r in reserved}:
|
||||
raise CallError("reserved_column", f"{what}「{column}」和结果表的列重名(结果表要用 {'、'.join(reserved)} 这几个名字):"
|
||||
f"在 SQL 里给它换个别名再分析,比如 AS {column}_name")
|
||||
|
||||
|
||||
def written_places(raw: Any) -> int:
|
||||
"""快照里一格写到第几位小数(headline 是取整过的,按它的书写精度容差):数字文本按字面,浮点按最短写法,整数为 0。"""
|
||||
"""快照里一格写到第几位小数(headline 是取整过的,按它的书写精度容差):数字文本按字面,浮点按最短写法,整数为 0。是整数
|
||||
的浮点(SQLite 的 ROUND() 交回 2120.0)是取到个位的,也为 0——最短写法里的「.0」不是书写精度。"""
|
||||
if isinstance(raw, bool) or raw is None:
|
||||
return 0
|
||||
if isinstance(raw, float) and raw.is_integer():
|
||||
return 0
|
||||
text = repr(raw) if isinstance(raw, float) else str(raw).strip()
|
||||
if "e" in text.lower():
|
||||
return 0
|
||||
return len(text.split(".", 1)[1]) if "." in text else 0
|
||||
|
||||
|
||||
def closes(total: Fraction, cell: Fraction, raw: Any) -> bool:
|
||||
def closes(total: Fraction, cell: Fraction, raw: Any, *, exact: bool = False) -> bool:
|
||||
"""各段之和(或合起来的比率)和总数那一格对得上:十进制逐位相等;总数取整过的,差在它书写精度的半个末位以内也算;浮点
|
||||
算出来的差在最后几位的也算(相对误差 1e-9 以内,同 engine/compare._equal)。"""
|
||||
算出来的差在最后几位的也算(相对误差 1e-9 以内,同 engine/compare._equal)。exact:对的是上一层那一段算出来的精确值
|
||||
(下钻),没有取整,只容浮点的最后几位。"""
|
||||
if total == cell:
|
||||
return True
|
||||
gap = abs(total - cell)
|
||||
if gap <= Fraction(1, 2) / Fraction(10) ** written_places(raw):
|
||||
if not exact and gap <= Fraction(1, 2) / Fraction(10) ** written_places(raw):
|
||||
return True
|
||||
scale = max(abs(total), abs(cell))
|
||||
return scale != 0 and gap <= scale * Fraction(1, 10 ** 9)
|
||||
@@ -148,4 +160,4 @@ def sign_word(value: Fraction | float | int) -> str | None:
|
||||
|
||||
|
||||
__all__ = ["DERIVED_PARAMS", "Env", "build", "closes", "column_label", "column_types", "label_for", "need", "refuse",
|
||||
"sign_word", "today_param", "written_places"]
|
||||
"reserved_check", "sign_word", "today_param", "written_places"]
|
||||
|
||||
@@ -3,11 +3,12 @@
|
||||
|
||||
五项检查,每项 pass / warn / fail / unknown(unknown 只写「核对不了」,不判不可比):
|
||||
|
||||
- 同口径:两份 SQL 一字不差 → pass;规范化(data/sql_semantics.normalize_sql)后只差字面量:把时间条件的界(data/coverage
|
||||
认出的时间谓词的界,整个表达式)和像日期的字面量换成同一个占位再比,一样就是只差日期 → pass,不一样就是比的两段不是
|
||||
两期 → fail「两边除了日期,筛选的还不一样」;规范形式不同交给口径账本的 explain_sql_diff,等价 → pass,只差时间范围、排序
|
||||
→ pass,否则 fail,写出差在哪些方面(只写方面,不写 SQL 原文里的值)。两份来自不同的数据源 fail。同一份快照里按期次分组
|
||||
的天然同口径。
|
||||
- 同口径:两份 SQL 一字不差 → pass;否则两边都去掉最外层的 ORDER BY(没有 LIMIT 时排序不改结果),把时间条件的界(data/
|
||||
coverage 认出的时间谓词的界,整个表达式:字面量、按运行当天现算的都算)和像日期的字面量换成同一个占位,再规范化
|
||||
(data/sql_semantics.normalize_sql)比:一样就是只差日期 → pass;只差字面量就是比的两段不是两期 → fail「两边除了日期,
|
||||
筛选的还不一样」;还不一样交给口径账本的 explain_sql_diff(同样去掉最外层的 ORDER BY,现算的界换成按先后编号的日期字面量:
|
||||
它只认字面量的时间范围),等价 → pass,只差时间范围、排序 → pass,否则 fail,写出差在哪些方面(只写方面,不写 SQL 原文里
|
||||
的值)。两份来自不同的数据源 fail。同一份快照里按期次分组的天然同口径。
|
||||
- 窗口长度:用 data/coverage 的 filters 取业务日期列(目录的 business_date;没写就取唯一那一列时间谓词)上的区间,界是字面
|
||||
量的直接解析,按运行当天现算的在库里算(Book 的 bound,记进 probes);算两边天数。整月对整月、整季对整季算一样长。
|
||||
不相等:度量是流量 fail,比率、存量、没标的 warn。按期次分组的:两期粒度相同,一样长。
|
||||
@@ -70,7 +71,8 @@ class Span:
|
||||
table: str | None = None
|
||||
column: str | None = None
|
||||
sql_window: Any = None
|
||||
unit: str | None = None # 这一段是不是整月、整季、整年(calendar_unit),按期次时是粒度
|
||||
unit: str | None = None # 这一段是不是整月、整季、整年(calendar_unit),按期次时是粒度(不是整期的为 None)
|
||||
short: int | None = None # 按期次的这一期不到整期时有几天(periods.short_days:跨年被切开的周)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -93,14 +95,26 @@ class Outcome:
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _unordered(sql: str, read: str | None) -> Any:
|
||||
"""解析成语法树,去掉最外层的 ORDER BY(同一层有 LIMIT / OFFSET 时排序决定取哪几行,不去)。解析不了为 None。"""
|
||||
from app.data import coverage
|
||||
|
||||
tree = coverage.parse_sql(sql, read)
|
||||
if tree is not None and tree.args.get("order") is not None and not tree.args.get("limit") \
|
||||
and not tree.args.get("offset"):
|
||||
tree.set("order", None)
|
||||
return tree
|
||||
|
||||
|
||||
def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None:
|
||||
"""时间条件的界、像日期的字面量换成同一个占位之后的规范形式(文本 + 剩下的字面量)。解析不了为 None。"""
|
||||
"""去掉最外层的 ORDER BY、时间条件的界和像日期的字面量换成同一个占位之后的规范形式(文本 + 剩下的字面量)。解析不了为
|
||||
None。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.data import coverage
|
||||
from app.data.sql_semantics import normalize_sql
|
||||
|
||||
tree = coverage.parse_sql(sql, read)
|
||||
tree = _unordered(sql, read)
|
||||
if tree is None:
|
||||
return None
|
||||
for flt in coverage.filters(tree):
|
||||
@@ -116,6 +130,28 @@ def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None:
|
||||
return normalized.text, normalized.literals
|
||||
|
||||
|
||||
def _for_diff(sql: str, read: str | None) -> str:
|
||||
"""交给 explain_sql_diff 的写法:去掉最外层的 ORDER BY;按运行当天现算的界换成按先后编号的日期字面量(它只认字面量的
|
||||
时间范围,现算的界写法不一样会落进兜底的「写法不同」)。解析不了的原样交。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.data import coverage
|
||||
|
||||
tree = _unordered(sql, read)
|
||||
if tree is None:
|
||||
return sql
|
||||
n = 0
|
||||
for flt in coverage.filters(tree):
|
||||
for _, node, _ in flt.conds:
|
||||
if node.parent is not None and not coverage.literal_bound(node)[0]:
|
||||
n += 1
|
||||
node.replace(exp.Literal.string(f"1900-01-{n:02d}"))
|
||||
try:
|
||||
return tree.sql(dialect=read)
|
||||
except Exception: # noqa: BLE001 - 写不回去就交原文
|
||||
return sql
|
||||
|
||||
|
||||
def same_query(base: Snap, target: Snap) -> Check:
|
||||
from app.data.sql_semantics import explain_sql_diff, normalize_sql
|
||||
|
||||
@@ -132,17 +168,17 @@ def same_query(base: Snap, target: Snap) -> Check:
|
||||
if na.parse_error or nb.parse_error:
|
||||
check.reasons = ["SQL 解析不了"]
|
||||
return check
|
||||
if na.text == nb.text:
|
||||
ua, ub = _undated(a, read), _undated(b, read)
|
||||
if ua is None or ub is None:
|
||||
return check
|
||||
if ua == ub:
|
||||
check.status = "pass"
|
||||
else:
|
||||
check.status, check.reasons = "fail", ["两边除了日期,筛选的还不一样"]
|
||||
ua, ub = _undated(a, read), _undated(b, read)
|
||||
if ua is None or ub is None:
|
||||
return check
|
||||
if ua == ub:
|
||||
check.status = "pass"
|
||||
return check
|
||||
if ua[0] == ub[0]:
|
||||
check.status, check.reasons = "fail", ["两边除了日期,筛选的还不一样"]
|
||||
return check
|
||||
try:
|
||||
diff = explain_sql_diff(a, b, read)
|
||||
diff = explain_sql_diff(_for_diff(a, read), _for_diff(b, read), read)
|
||||
except Exception: # noqa: BLE001 - 比不了就当核对不了
|
||||
return check
|
||||
aspects = [d.aspect for d in diff.items]
|
||||
@@ -238,7 +274,9 @@ def period_span(label: str, snap: Snap, period: Any, item: pd.Period, grain: str
|
||||
"""按期次分组的一份快照里的一期:这一期覆盖的日子,加上 SQL 在期次那一列上的时间条件。"""
|
||||
from app.data import coverage
|
||||
|
||||
span = Span(label, snap, item.start, item.end, unit=grain)
|
||||
span = Span(label, snap, item.start, item.end, unit=grain, short=pd.short_days(item, grain))
|
||||
if span.short:
|
||||
span.unit = None
|
||||
found = date_source(snap.sql, period)
|
||||
if found is None:
|
||||
return span
|
||||
@@ -268,10 +306,13 @@ def _range(span: Span, book: Book) -> tuple[bool, date | None, date | None, bool
|
||||
|
||||
|
||||
def span_coverage(span: Span, today: date, book: Book) -> tuple[str, str | None]:
|
||||
"""这一段(窗口或一期)的数据齐不齐:(pass / short / unknown, 不齐的那一句)。没过完(终点不早于运行当天:当天的数据还没
|
||||
到齐)、SQL 的时间条件截掉了一部分、表里的数据没到(最晚早于终点)或来得晚(最早晚于起点),都算不齐。"""
|
||||
"""这一段(窗口或一期)的数据齐不齐:(pass / short / unknown, 不齐的那一句)。这一期本身不到整期(跨年被切开的周)、没过完
|
||||
(终点不早于运行当天:当天的数据还没到齐)、SQL 的时间条件截掉了一部分、表里的数据没到(最晚早于终点)或来得晚(最早晚于
|
||||
起点),都算不齐。"""
|
||||
if span.start is None or span.end is None:
|
||||
return "unknown", None
|
||||
if span.short:
|
||||
return "short", f"{span.label}只有 {span.start.isoformat()} 到 {span.end.isoformat()} 这 {span.short} 天,不是整周"
|
||||
lo, hi = span.start, span.end
|
||||
window = span.sql_window
|
||||
if window is not None:
|
||||
@@ -417,20 +458,32 @@ def compare_queries(base: Snap, target: Snap, measure: str | None, facts: Facts,
|
||||
|
||||
def compare_periods(snap: Snap, period: str, base: pd.Period, target: pd.Period, grain: str, measure: str | None,
|
||||
facts: Facts, book: Book, today: date) -> Outcome:
|
||||
"""同一份快照里按期次分组的两期:天然同口径;窗口按期次的粒度;整期看这两期各自齐不齐。"""
|
||||
"""同一份快照里按期次分组的两期:天然同口径;窗口按期次的粒度(有一期不到整期的按实际天数比,星期构成也照算);整期看
|
||||
这两期各自齐不齐。期次不知道是哪几天的(序号、认不出算法的周号)窗口、整期、星期构成都核对不了。"""
|
||||
same = Check("same_query", status="pass")
|
||||
b_span = period_span("基期", snap, period, base, grain, measure, facts, book)
|
||||
t_span = period_span("本期", snap, period, target, grain, measure, facts, book)
|
||||
flow = bool(measure) and facts.measure(snap, measure)[0] == "flow" # type: ignore[arg-type]
|
||||
if grain == "seq":
|
||||
whole = not (b_span.short or t_span.short)
|
||||
if grain == "seq" or base.start is None or target.start is None:
|
||||
checks = [same, Check("window"), Check("complete"), Check("weekdays"), _data_check([("", snap)], measure)]
|
||||
else:
|
||||
checks = [same, _window_check(b_span, t_span, flow, grouped=True),
|
||||
checks = [same, _window_check(b_span, t_span, flow, grouped=whole),
|
||||
_complete_check([b_span, t_span], today, book, flow),
|
||||
_weekday_check(b_span, t_span, grain=grain), _data_check([("", snap)], measure)]
|
||||
_weekday_check(b_span, t_span, grain=grain if whole else None), _data_check([("", snap)], measure)]
|
||||
return _decide(checks, b_span, t_span)
|
||||
|
||||
|
||||
def period_complete(snap: Snap, period: str, item: pd.Period, grain: str, measure: str | None, facts: Facts,
|
||||
book: Book, today: date) -> tuple[str, str | None, str | None]:
|
||||
"""按期次分组的一份快照里本期齐不齐(波动基线、趋势、单份快照的变化拆解共用):(pass / short / unknown, 不齐的那一句,
|
||||
核对不了时附带说明的那一句)。期次没有日期(序号)、周号认不出算法的,起止不知道,一律核对不了,不判齐。"""
|
||||
if item.start is None:
|
||||
return "unknown", None, ph.UNKNOWN_WEEKS if grain == "week" else ph.NO_DATES
|
||||
status, why = span_coverage(period_span("本期", snap, period, item, grain, measure, facts, book), today, book)
|
||||
return status, why, ph.UNVERIFIED_COMPLETE if status == "unknown" else None
|
||||
|
||||
|
||||
def table_of(outcome: Outcome) -> dict[str, Any]:
|
||||
"""检查表(结论的 table):检查项、结果、基期、本期、说明。"""
|
||||
rows = [[CHECK_NAMES[c.id], STATUS_NAMES[c.status], c.base, c.target, ";".join(c.reasons) or None]
|
||||
@@ -479,8 +532,9 @@ def comparable(raw: Mapping[str, Any], env: Any) -> Any:
|
||||
params: dict[str, Any] = {}
|
||||
two = raw.get("base_query") is not None or raw.get("target_query") is not None
|
||||
if two:
|
||||
base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号")
|
||||
target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号")
|
||||
base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号", role="base")
|
||||
target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号",
|
||||
role="target")
|
||||
if base is target:
|
||||
from app.analysis.decompose import SAME_QUERY
|
||||
|
||||
@@ -511,9 +565,11 @@ def comparable(raw: Mapping[str, Any], env: Any) -> Any:
|
||||
for side, what in (("base", "基期"), ("target", "本期")):
|
||||
if not rows[side]:
|
||||
raise CallError("period_not_found", f"期次列「{period}」里没有「{params[side]}」:{what}要写 {target.ref} "
|
||||
"里有的期次值")
|
||||
"里有的期次值",
|
||||
reason=f"期次列「{period}」里没有{what}「{ph.key_text([params[side]])}」")
|
||||
env.inputs.use(target, "data", [period, *([measure] if measure else [])], rows["base"] + rows["target"])
|
||||
parsed = pd.parse(values, grain)
|
||||
parsed = pd.parse(values, grain, weeks=pd.week_scheme(target.sql, period, target.dialect)) if grain \
|
||||
else pd.parse_compared(values, target.sql, period, target.dialect)
|
||||
if parsed is None:
|
||||
item_b = item_t = pd.Period(None, 0, None, None)
|
||||
grain = "seq"
|
||||
@@ -538,5 +594,5 @@ def _only_number(snap: Snap) -> str | None:
|
||||
|
||||
|
||||
__all__ = ["CHECKS", "CHECK_NAMES", "Check", "Outcome", "STATUS_NAMES", "Span", "comparable", "compare_periods",
|
||||
"compare_queries", "period_span", "reasons", "same_query", "span_coverage", "sql_span", "table_of",
|
||||
"values_of"]
|
||||
"compare_queries", "period_complete", "period_span", "reasons", "same_query", "span_coverage", "sql_span",
|
||||
"table_of", "values_of"]
|
||||
|
||||
@@ -83,16 +83,20 @@ VERDICTS: dict[str, tuple[str, ...]] = {
|
||||
"differ": ("differs", "no_evidence"),
|
||||
"comparable": ("comparable", "conditional", "incomparable"),
|
||||
}
|
||||
#: 算不了的原因(细案 01 §8.2)。算不了也记成结论:写作者可以照实写「历史只有 5 周,判断不了是否异常」
|
||||
#: 算不了的原因(细案 01 §8.2)。算不了也记成结论:写作者可以照实写「历史只有 5 周,判断不了是否异常」。最后两种是分析卡
|
||||
#: 里随数据变化的调用错误记成的(analysis.run 的 refuse_data_errors,见 app/analysis 的 DATA_ERRORS):parent_refused 上一层
|
||||
#: 没做成;input_unavailable 这次的数据里没有要用的那一项(下钻的段、期次、总数那一格、要比的组……,reason 写是哪一项、哪个值)。
|
||||
#: untestable:差异检验一对都拿不到 p 值(两组各 1 个样本、样本量为 0……)
|
||||
REASONS: tuple[str, ...] = (
|
||||
"truncated", "too_few_segments", "history_short", "partial_target", "target_gap", "series_gap",
|
||||
"period_unreadable", "zero_base", "non_positive", "incomparable_sides",
|
||||
"period_unreadable", "zero_base", "non_positive", "incomparable_sides", "parent_refused", "input_unavailable",
|
||||
"untestable",
|
||||
)
|
||||
#: 附带说明的种类(caveats[].code)
|
||||
CAVEATS: tuple[str, ...] = (
|
||||
"offset", "simpson", "small_denominator", "record_high", "record_low", "weekday_effect",
|
||||
"multiple_comparisons", "outlier_sensitive", "small_sample", "measure_unlabeled", "null_values",
|
||||
"headline_basis", "comparability",
|
||||
"headline_basis", "comparability", "partial_periods", "untested",
|
||||
)
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -163,7 +167,8 @@ LABELS: dict[str, str] = {
|
||||
|
||||
def settings_problem(values: Mapping[str, Any]) -> str | None:
|
||||
"""一组分析设置(或分析卡上的覆盖)哪里不对:不认识的键、类型不对、超出范围、k_alert 不大于 k_warn、
|
||||
max_history 小于 min_history。都对返回 None。说法给人看。"""
|
||||
max_history 小于 min_history、concentration 小于 main_share(一段占了「主要」那么多还够不上「集中」,判定语会说「没有哪一段
|
||||
占到」一个比它小的数)。没写的项按缺省补齐了比。都对返回 None。说法给人看(键的叫法取 LABELS)。"""
|
||||
for key, value in values.items():
|
||||
if key not in RANGES:
|
||||
return f"不认识的分析设置:{key}"
|
||||
@@ -180,6 +185,8 @@ def settings_problem(values: Mapping[str, Any]) -> str | None:
|
||||
return f"{LABELS['k_alert']}要大于{LABELS['k_warn']}"
|
||||
if merged["max_history"] < merged["min_history"]:
|
||||
return f"{LABELS['max_history']}不能小于{LABELS['min_history']}"
|
||||
if merged["concentration"] < merged["main_share"]:
|
||||
return f"{LABELS['concentration']}不能小于{LABELS['main_share']}"
|
||||
return None
|
||||
|
||||
|
||||
@@ -305,12 +312,16 @@ class AnalysisResult(TypedDict, total=False):
|
||||
class CallError(Exception):
|
||||
"""调用错误(细案 01 §8.1):列不存在、列被遮罩、角色和目录度量类型冲突、同一键同一期多行、期次不在列里、乘积
|
||||
不闭合、parent 不对、下钻超深、这个 Agent 这次执行的分析次数用完(max_calls 按每个 Agent 每次执行计)。不记结论,message
|
||||
交回 agent(一种说法,告诉它怎么改)。"""
|
||||
交回 agent(一种说法,告诉它怎么改)。
|
||||
|
||||
def __init__(self, code: str, message: str) -> None:
|
||||
随数据变化的那几种(app/analysis 的 DATA_ERRORS:这期没有要用的期次、段、组,上一层没做成……)另带 reason:照实说是哪一项、
|
||||
哪个值,不带改法。分析卡(模板定死了调用)按它记成算不了的结论,见 analysis.run 的 refuse_data_errors。"""
|
||||
|
||||
def __init__(self, code: str, message: str, *, reason: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.reason = reason
|
||||
|
||||
|
||||
#: 现查一列的实际范围:(数据源名, 表, 列) → Probe;查不了返回 None(算子写「核对不了」,不判不可比)
|
||||
|
||||
@@ -2,9 +2,10 @@
|
||||
|
||||
**参数**(AnalysisRequest.params;列名不分大小写,按快照认):
|
||||
|
||||
- 一份快照:query(编号)、period(期次列)、base / target(期次列里的两个值);或两份快照:base_query、target_query(各自
|
||||
只有维度列和度量列,先做可比性检查:口径结构不同的不拆,算不了 incomparable_sides;其余不可比的照样算,判定后附上
|
||||
可比性结论)。
|
||||
- 一份快照:query(编号)、period(期次列)、base / target(期次列里的两个值),本期齐不齐同波动基线那一套(comparable.
|
||||
period_complete:不到整期、没过完、数据没到 → partial_target;核对不了的附说明,只在证据面板);或两份快照:base_query、
|
||||
target_query(各自只有维度列和度量列,先做可比性检查:口径结构不同的不拆,算不了 incomparable_sides;其余不可比的照样算,
|
||||
判定后附上可比性结论)。
|
||||
- additive:by(维度列,1–2 列)、measure(度量列);可选 headline {base, target}(基期、本期总数各一格,如 Q1.r1.gmv,用来
|
||||
核对闭合)、parent {of: A3, key: [华东]}(下钻:这一层各段之和要等于 A3 里华东那一段)。
|
||||
- ratio:by、numerator(分子列)、denominator(分母列);可选 headline {base, target}(整体比率的两格)、parent(上一层的比率
|
||||
@@ -26,8 +27,8 @@ from typing import Any
|
||||
|
||||
from app.analysis import comparable as cmp
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis.common import (Env, build, closes, column_label, column_types, label_for, need, refuse, sign_word,
|
||||
today_param)
|
||||
from app.analysis.common import (Env, build, closes, column_label, column_types, label_for, need, refuse,
|
||||
reserved_check, sign_word, today_param)
|
||||
from app.analysis.contract import CallError
|
||||
from app.analysis.inputs import Snap, as_list, hashable, same_value
|
||||
from app.analysis.numbers import canon, to_fraction
|
||||
@@ -41,6 +42,22 @@ SAME_QUERY = "基期和本期是同一份查询:两期各给一份查询,或
|
||||
#: 乘积闭合的最小相对误差
|
||||
_FACTOR_REL = Fraction(1, 10 ** 6)
|
||||
_NOT_ADDITIVE = {"ratio": "比率", "identifier": "标识", "status": "状态", "attribute": "属性"}
|
||||
#: 各模式的结论里记的阈值(下钻时另加 max_drill_depth)
|
||||
USED: dict[str, tuple[str, ...]] = {
|
||||
"additive": ("main_share", "concentration", "offset_ratio", "flat_pct", "table_top_k"),
|
||||
"ratio": ("main_share", "flat_pct", "min_denominator", "table_top_k"),
|
||||
"factor": ("main_share", "flat_pct"),
|
||||
}
|
||||
|
||||
|
||||
def refusal_basis(mode: str, raw: Mapping[str, Any], settings: Mapping[str, Any]) -> tuple[list[str], str]:
|
||||
"""还没算就算不了时(分析卡把随数据变化的调用错误记成算不了的结论)结论里记的阈值和判定标准:同这个模式自己算不了时
|
||||
的那一份。"""
|
||||
used = list(USED[mode]) + (["max_drill_depth"] if mode != "factor" and raw.get("parent") is not None else [])
|
||||
if mode == "additive":
|
||||
summed = raw.get("headline") is None and raw.get("parent") is None
|
||||
return used, ph.criterion_additive(settings, top_k_max=None, summed=summed)
|
||||
return used, ph.criterion_ratio(settings) if mode == "ratio" else ph.criterion_factor(settings)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -65,14 +82,16 @@ def _key_strings(key: Sequence[Any]) -> list[str]:
|
||||
return [ph.key_text([k]) for k in key]
|
||||
|
||||
|
||||
def _sides(raw: Mapping[str, Any], env: Env, roles: Sequence[tuple[str, str]], *, by_needed: bool = True
|
||||
) -> tuple[dict[str, Any], Snap, Snap, dict[str, str], list[str]]:
|
||||
"""一份或两份快照:(规范化的参数, 基期快照, 本期快照, {角色: 列名}, 维度列)。roles 是 [(参数名, 角色的叫法)]。"""
|
||||
def _sides(raw: Mapping[str, Any], env: Env, roles: Sequence[tuple[str, str]], *, by_needed: bool = True,
|
||||
reserved: Sequence[str] = ()) -> tuple[dict[str, Any], Snap, Snap, dict[str, str], list[str]]:
|
||||
"""一份或两份快照:(规范化的参数, 基期快照, 本期快照, {角色: 列名}, 维度列)。roles 是 [(参数名, 角色的叫法)];reserved 是
|
||||
结果表固定的列名,维度列不能叫这些。"""
|
||||
params: dict[str, Any] = {}
|
||||
two = raw.get("base_query") is not None or raw.get("target_query") is not None
|
||||
if two:
|
||||
base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号")
|
||||
target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号")
|
||||
base = env.inputs.snap(need(raw, "base_query", "基期的查询编号"), "基期的查询编号", role="base")
|
||||
target = env.inputs.snap(need(raw, "target_query", "本期的查询编号"), "本期的查询编号",
|
||||
role="target")
|
||||
if base is target:
|
||||
raise CallError("same_query", SAME_QUERY)
|
||||
params.update(base_query=base.ref, target_query=target.ref)
|
||||
@@ -88,6 +107,8 @@ def _sides(raw: Mapping[str, Any], env: Env, roles: Sequence[tuple[str, str]], *
|
||||
if two:
|
||||
for c in by:
|
||||
base.column(c, "维度列")
|
||||
for c in by:
|
||||
reserved_check(c, reserved, "维度列")
|
||||
params["by"] = by
|
||||
cols: dict[str, str] = {}
|
||||
for key, what in roles:
|
||||
@@ -116,7 +137,8 @@ def _period_rows(snap: Snap, period: str, wanted: Any, what: str) -> list[int]:
|
||||
if not rows:
|
||||
values = list(dict.fromkeys(ph.key_text([row[i]]) for row in snap.rows if i < len(row)))
|
||||
raise CallError("period_not_found", f"期次列「{period}」里没有「{wanted}」:{what}要写 {snap.ref} 里有的期次值,"
|
||||
f"比如 {'、'.join(values[:6])}")
|
||||
f"比如 {'、'.join(values[:6])}",
|
||||
reason=f"期次列「{period}」里没有{what}「{ph.key_text([wanted])}」")
|
||||
return rows
|
||||
|
||||
|
||||
@@ -181,8 +203,12 @@ def _parent(raw: Mapping[str, Any], env: Env, mode: str, dims: int) -> tuple[dic
|
||||
if found.get("op") != "decompose" or found.get("mode") != mode:
|
||||
raise CallError("parent_not_decompose", f"{name} 不是{'按维度' if mode == 'additive' else '比率'}的变化拆解,"
|
||||
"下钻要接在同一种拆解后面")
|
||||
# 上一层先记进输入:它没做成、这期没有那一段时(分析卡记成算不了的结论),复算也要按内容哈希取回它、走到同一处
|
||||
env.inputs.parent(name, content_hash(canonical_json(found)))
|
||||
if found.get("status") != "ok" or not isinstance(found.get("table"), Mapping):
|
||||
raise CallError("parent_refused", f"{name} 没有做成,不能在它下面下钻")
|
||||
why = f"({found['reason']})" if isinstance(found.get("reason"), str) and found["reason"] else ""
|
||||
raise CallError("parent_refused", f"{name} 没有做成,不能在它下面下钻",
|
||||
reason=f"上一层「{name}」没有做成{why},不能在它下面下钻")
|
||||
depth = int((found.get("params") or {}).get("depth") or 0) + 1
|
||||
limit = int(env.settings["max_drill_depth"])
|
||||
if depth > limit:
|
||||
@@ -196,8 +222,8 @@ def _parent(raw: Mapping[str, Any], env: Env, mode: str, dims: int) -> tuple[dic
|
||||
hit = next((row for row in table.get("rows") or []
|
||||
if all(same_value(k, v) for k, v in zip(key, row[:dim_count]))), None)
|
||||
if hit is None:
|
||||
raise CallError("parent_key_missing", f"{name} 里没有「{ph.key_text(key)}」这一段:写 {name} 结果表里的段")
|
||||
env.inputs.parent(name, content_hash(canonical_json(found)))
|
||||
raise CallError("parent_key_missing", f"{name} 里没有「{ph.key_text(key)}」这一段:写 {name} 结果表里的段",
|
||||
reason=f"上一层「{name}」里没有「{ph.key_text(key)}」这一段")
|
||||
return {"of": name, "key": key}, dict(zip(columns, hit)), depth
|
||||
|
||||
|
||||
@@ -220,7 +246,8 @@ def _headline(raw: Mapping[str, Any], env: Env) -> tuple[dict[str, str] | None,
|
||||
for side, what in (("base", "基期总数"), ("target", "本期总数")):
|
||||
snap, row, col, value = env.inputs.cell(spec[side], f"headline_{side}", what)
|
||||
if value is None:
|
||||
raise CallError("headline_empty", f"{what}那一格({snap.ref}.r{row}.{col})是空的:换一格有值的")
|
||||
raise CallError("headline_empty", f"{what}那一格({snap.ref}.r{row}.{col})是空的:换一格有值的",
|
||||
reason=f"{what}那一格({snap.ref}.r{row}.{col})是空的")
|
||||
out[side] = (value, snap.cell(row, col))
|
||||
refs[side] = f"{snap.ref}.r{row}.{col}"
|
||||
return refs, out
|
||||
@@ -249,6 +276,30 @@ def _comparability(env: Env, params: dict[str, Any], base: Snap, target: Snap, m
|
||||
return outcome, outcome.structure
|
||||
|
||||
|
||||
def _period_complete(env: Env, params: dict[str, Any], snap: Snap, measure: str | None
|
||||
) -> tuple[str | None, dict[str, str] | None]:
|
||||
"""一份快照按期次分组时本期齐不齐(同波动基线那一套,comparable.period_complete;期次按 periods.parse_compared 认):
|
||||
(不齐时 partial_target 的原因, 核对不了时的附带说明)。两份快照的在可比性检查里核过整期,不在这里核。"""
|
||||
from app.analysis import periods as pd
|
||||
|
||||
if "base_query" in params:
|
||||
return None, None
|
||||
params["today"] = today_param(env)
|
||||
period = params["period"]
|
||||
i = snap.index(period)
|
||||
values = [row[i] if i < len(row) else None for row in snap.rows]
|
||||
parsed = pd.parse_compared(values, snap.sql, period, snap.dialect)
|
||||
if parsed is None:
|
||||
return None, ph.caveat("comparability", ph.NO_DATES)
|
||||
items, grain = parsed
|
||||
target = next(item for item, v in zip(items, values) if same_value(params["target"], v))
|
||||
status, why, note = cmp.period_complete(snap, period, target, grain, measure, env.facts, env.book,
|
||||
env.today) # type: ignore[arg-type]
|
||||
if status == "short":
|
||||
return f"{why},和完整的{pd.PER_WORD[grain]}比没有意义", None
|
||||
return None, ph.caveat("comparability", note) if status == "unknown" and note else None
|
||||
|
||||
|
||||
def _share_rank(segments: Sequence[Any], share: Any) -> list[Any]:
|
||||
"""按 share 从大到小(一样大的按先出现的)。"""
|
||||
return sorted(segments, key=lambda s: -share(s))
|
||||
@@ -260,8 +311,8 @@ def _share_rank(segments: Sequence[Any], share: Any) -> list[Any]:
|
||||
|
||||
|
||||
def additive(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["main_share", "concentration", "offset_ratio", "flat_pct", "table_top_k"]
|
||||
params, bs, ts, cols, by = _sides(raw, env, [("measure", "度量列")])
|
||||
used = list(USED["additive"])
|
||||
params, bs, ts, cols, by = _sides(raw, env, [("measure", "度量列")], reserved=_FIXED_ADDITIVE)
|
||||
measure = cols["measure"]
|
||||
label = label_for(env, raw.get("label"), ts, measure)
|
||||
if raw.get("label") is not None:
|
||||
@@ -297,9 +348,16 @@ def additive(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
if structure is not None:
|
||||
return refuse(env, params, used, "incomparable_sides", f"{structure},不是同一个量的两期,不拆",
|
||||
criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats)
|
||||
partial, note = _period_complete(env, params, ts, measure)
|
||||
if partial is not None:
|
||||
return refuse(env, params, used, "partial_target", partial,
|
||||
criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats)
|
||||
if note is not None:
|
||||
caveats.append(note)
|
||||
segs = _segments(side_b, side_t, Fraction(0))
|
||||
if len(segs) < 2:
|
||||
return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(segs)} 段,至少要 2 段才拆得开",
|
||||
return refuse(env, params, used, "too_few_segments",
|
||||
ph.say("按", ph.dims_text(dims), f"只有 {len(segs)} 段,至少要 2 段才拆得开"),
|
||||
criterion=ph.criterion_additive(env.settings, top_k_max=None, **criterion_args), caveats=caveats)
|
||||
return _additive_result(env, params, used, segs, by, dims, label, heads, parent_row, outcome, caveats, summed,
|
||||
side_b, side_t)
|
||||
@@ -321,7 +379,7 @@ def _additive_result(env: Env, params: dict[str, Any], used: list[str], segs: li
|
||||
gaps[side] = heads[side][0] - total
|
||||
if parent_row is not None:
|
||||
expect = to_fraction(parent_row.get(side))
|
||||
if expect is None or not closes(total, expect, parent_row.get(side)):
|
||||
if expect is None or not closes(total, expect, None, exact=True):
|
||||
gaps.setdefault(side, (expect or Fraction(0)) - total)
|
||||
closed = not gaps
|
||||
deltas = {x.key: x.target - x.base for x in segs}
|
||||
@@ -408,7 +466,9 @@ def _additive_result(env: Env, params: dict[str, Any], used: list[str], segs: li
|
||||
values["gap"] = canon(max(gaps.values(), key=abs))
|
||||
verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects],
|
||||
"direction": sign_word(delta) if code in ("driver", "concentrated", "dispersed") else None}
|
||||
phrase = ph.with_caveats(main, caveats, skip=("offset",) if code == "flat" else ())
|
||||
# 一份快照时 comparability 只会是本期核对不了的那句(同波动基线,只在证据面板);两份快照的可比性结论照旧进判定语
|
||||
skip = (("offset",) if code == "flat" else ()) + (("comparability",) if outcome is None else ())
|
||||
phrase = ph.with_caveats(main, caveats, skip=skip)
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict,
|
||||
criterion=ph.criterion_additive(s, top_k_max=k_max, summed=summed), phrase=phrase, caveats=caveats)
|
||||
|
||||
@@ -419,8 +479,9 @@ def _additive_result(env: Env, params: dict[str, Any], used: list[str], segs: li
|
||||
|
||||
|
||||
def ratio(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["main_share", "flat_pct", "min_denominator", "table_top_k"]
|
||||
params, bs, ts, cols, by = _sides(raw, env, [("numerator", "分子列"), ("denominator", "分母列")])
|
||||
used = list(USED["ratio"])
|
||||
params, bs, ts, cols, by = _sides(raw, env, [("numerator", "分子列"), ("denominator", "分母列")],
|
||||
reserved=_FIXED_RATIO)
|
||||
num, den = cols["numerator"], cols["denominator"]
|
||||
num_label, den_label = column_label(env, ts, num), column_label(env, ts, den)
|
||||
label = ph.pick_label(raw.get("label"), ph.say(num_label, "与", den_label, "之比"))
|
||||
@@ -459,10 +520,14 @@ def ratio(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
outcome, structure = _comparability(env, params, bs, ts, den)
|
||||
if structure is not None:
|
||||
return refuse(env, params, used, "incomparable_sides", f"{structure},不是同一个量的两期,不拆", criterion=criterion)
|
||||
partial, note = _period_complete(env, params, ts, den)
|
||||
if partial is not None:
|
||||
return refuse(env, params, used, "partial_target", partial, criterion=criterion)
|
||||
segs = _segments(side_b, side_t, (Fraction(0), Fraction(0)))
|
||||
live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0]
|
||||
if len(live) < 2:
|
||||
return refuse(env, params, used, "too_few_segments", f"按{ph.dims_text(dims)}只有 {len(live)} 段,至少要 2 段才拆得开",
|
||||
return refuse(env, params, used, "too_few_segments",
|
||||
ph.say("按", ph.dims_text(dims), f"只有 {len(live)} 段,至少要 2 段才拆得开"),
|
||||
criterion=criterion)
|
||||
for when, i in (("基期", 0), ("本期", 1)):
|
||||
if sum(((x.base, x.target)[i][1] for x in live), Fraction(0)) == 0:
|
||||
@@ -475,7 +540,7 @@ def ratio(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
ph.say(ph.key_text(x.key), f"{when}的", den_label, "为 0 而", num_label,
|
||||
"不为 0,比率拆不了"), criterion=criterion)
|
||||
return _ratio_result(env, params, used, segs, by, dims, label, num_label, den_label, heads, parent_row, outcome,
|
||||
criterion, target_snap=ts)
|
||||
criterion, target_snap=ts, note=note)
|
||||
|
||||
|
||||
def ratio_effects(segs: Sequence[tuple[Fraction, Fraction, Fraction, Fraction]]
|
||||
@@ -504,7 +569,8 @@ def ratio_effects(segs: Sequence[tuple[Fraction, Fraction, Fraction, Fraction]]
|
||||
def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[Segment], by: list[str],
|
||||
dims: list[str], label: str, num_label: str, den_label: str,
|
||||
heads: Mapping[str, tuple[Fraction, Any]], parent_row: Mapping[str, Any] | None,
|
||||
outcome: cmp.Outcome | None, criterion: str, *, target_snap: Snap) -> Any:
|
||||
outcome: cmp.Outcome | None, criterion: str, *, target_snap: Snap,
|
||||
note: Mapping[str, str] | None = None) -> Any:
|
||||
main_share, flat_pct = env.threshold("main_share"), env.threshold("flat_pct")
|
||||
min_den = env.threshold("min_denominator")
|
||||
live = [x for x in segs if x.base[1] != 0 or x.target[1] != 0]
|
||||
@@ -522,7 +588,7 @@ def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[
|
||||
"d_target": sum((x.target[1] for x in live), Fraction(0))}
|
||||
for col, total in sums.items():
|
||||
expect = to_fraction(parent_row.get(col))
|
||||
if expect is None or not closes(total, expect, parent_row.get(col)):
|
||||
if expect is None or not closes(total, expect, None, exact=True):
|
||||
gaps.setdefault("base" if col.endswith("base") else "target", (expect or Fraction(0)) - total)
|
||||
closed = not gaps
|
||||
mix_total = sum((e[4] for e in effects), Fraction(0))
|
||||
@@ -552,9 +618,11 @@ def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[
|
||||
against = sum(((by_key[x.key][0] + by_key[x.key][1]) / 2 for x in eligible
|
||||
if (by_key[x.key][3] - by_key[x.key][2]) * sign < 0), Fraction(0))
|
||||
if weight > 0 and against / weight >= SIMPSON_SHARE:
|
||||
# 判在结构变化上(mix_driven)时判定语正文已经说了结构变化、点了这一段,附带说明只说各段和整体反着变
|
||||
star = max(live, key=lambda x: sign * by_key[x.key][4])
|
||||
caveats.append(ph.caveat("simpson", ph.simpson_text(label, delta > 0, ph.key_text(star.key),
|
||||
by_key[star.key][0], by_key[star.key][1])))
|
||||
by_key[star.key][0], by_key[star.key][1],
|
||||
shift=code != "mix_driven")))
|
||||
if small:
|
||||
caveats.append(ph.caveat("small_denominator", ph.small_denominator_text(
|
||||
[ph.key_text(x.key) for x in small], den_label, int(env.settings["min_denominator"]))))
|
||||
@@ -564,6 +632,8 @@ def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[
|
||||
ph.say(num_label, "或", den_label))))
|
||||
if outcome is not None and outcome.verdict != "comparable":
|
||||
caveats.append(ph.caveat("comparability", outcome.phrase))
|
||||
if note is not None:
|
||||
caveats.append(dict(note))
|
||||
total_text = ph.ratio_change(R, R2, proportion=proportion)
|
||||
if code == "mix_driven":
|
||||
star = subjects[0]
|
||||
@@ -610,7 +680,8 @@ def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[
|
||||
verdict = {"code": code, "subjects": [_key_strings(x.key) for x in subjects],
|
||||
"direction": sign_word(delta) if code in ("mix_driven", "rate_driven", "both") else None}
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
|
||||
phrase=ph.with_caveats(main, caveats), caveats=caveats)
|
||||
phrase=ph.with_caveats(main, caveats, skip=("comparability",) if outcome is None else ()),
|
||||
caveats=caveats)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -635,7 +706,7 @@ def lmdi(base: Sequence[float], target: Sequence[float], m_base: float, m_target
|
||||
|
||||
|
||||
def factor(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["main_share", "flat_pct"]
|
||||
used = list(USED["factor"])
|
||||
params, bs, ts, cols, _ = _sides(raw, env, [("product", "乘积列")], by_needed=False)
|
||||
product = cols["product"]
|
||||
wanted = as_list(raw.get("factors"))
|
||||
@@ -672,7 +743,8 @@ def factor(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
for col, what in ((product, "乘积列"), *((c, "因子列") for c in factors)):
|
||||
value = snap.number(r, col, what)
|
||||
if value is None:
|
||||
raise CallError("empty_cell", f"{snap.ref} 第 {r} 行的「{col}」是空的:乘积、因子都要有值")
|
||||
raise CallError("empty_cell", f"{snap.ref} 第 {r} 行的「{col}」是空的:乘积、因子都要有值",
|
||||
reason=f"{'基期' if when == 'base' else '本期'}的「{col}」那一格是空的({snap.ref} 第 {r} 行)")
|
||||
row.append((value, snap.cell(r, col)))
|
||||
cells[when] = row
|
||||
if bs is ts:
|
||||
@@ -683,6 +755,9 @@ def factor(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
outcome, structure = _comparability(env, params, bs, ts, product)
|
||||
if structure is not None:
|
||||
return refuse(env, params, used, "incomparable_sides", f"{structure},不是同一个量的两期,不拆", criterion=criterion)
|
||||
partial, note = _period_complete(env, params, ts, product)
|
||||
if partial is not None:
|
||||
return refuse(env, params, used, "partial_target", partial, criterion=criterion)
|
||||
if any(v <= 0 for row in cells.values() for v, _ in row):
|
||||
return refuse(env, params, used, "non_positive", "乘积或因子里有不大于 0 的值,对数拆解要求全为正", criterion=criterion)
|
||||
for when, row in cells.items():
|
||||
@@ -706,6 +781,8 @@ def factor(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
caveats: list[dict[str, str]] = []
|
||||
if outcome is not None and outcome.verdict != "comparable":
|
||||
caveats.append(ph.caveat("comparability", outcome.phrase))
|
||||
if note is not None:
|
||||
caveats.append(note)
|
||||
total_text = ph.change(delta, M)
|
||||
top = max(range(len(factors)), key=lambda i: shares[i] if shares[i] is not None else -math.inf)
|
||||
if flat:
|
||||
@@ -733,8 +810,9 @@ def factor(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
"top_share": canon(shares[top]) if shares[top] is not None else None}
|
||||
verdict = {"code": code, "subjects": subjects, "direction": sign_word(delta) if code != "flat" else None}
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
|
||||
phrase=ph.with_caveats(main, caveats), caveats=caveats)
|
||||
phrase=ph.with_caveats(main, caveats, skip=("comparability",) if outcome is None else ()),
|
||||
caveats=caveats)
|
||||
|
||||
|
||||
__all__ = ["HEADLINE_SHAPE", "PARENT_SHAPE", "SAME_QUERY", "SIMPSON_SHARE", "additive", "factor", "lmdi", "ratio",
|
||||
"ratio_effects"]
|
||||
__all__ = ["HEADLINE_SHAPE", "PARENT_SHAPE", "SAME_QUERY", "SIMPSON_SHARE", "USED", "additive", "factor", "lmdi",
|
||||
"ratio", "ratio_effects", "refusal_basis"]
|
||||
|
||||
@@ -16,6 +16,10 @@
|
||||
按 A − B 的正负);否则 no_evidence。用了 Fisher(比率)或有一组样本数小于 min_n(均值)→ small_sample。判定语不出现「p 值」
|
||||
「显著性」:differs 说「差异站得住」,no_evidence 说「差异可能只是样本波动」,p 值、区间在表和证据面板里。
|
||||
|
||||
拿不到 p 值的一对(比率有一组样本量为 0;均值、逐行有一组不到 2 个样本,或均值、标准差为空)不算进比了几对、不参与校正,表里
|
||||
照实写哪一组怎么了;一对都拿不到时算不了(untestable),不硬判「可能只是样本波动」;只有一部分拿不到的,判定语后接一句
|
||||
「另有 n 对算不了」(untested)。
|
||||
|
||||
subjects 是判定说的那一对 [[A], [B]](只比一对时就是那一对,比了几对时是校正后最站得住的那一对;几对都不站得住时为空),
|
||||
direction 是 A 相对 B。
|
||||
"""
|
||||
@@ -36,6 +40,16 @@ from app.analysis.numbers import canon, to_fraction
|
||||
TESTS = {"z": "z 检验", "fisher": "Fisher 精确检验", "welch": "Welch t 检验"}
|
||||
|
||||
|
||||
def used_settings(mode: str | None) -> list[str]:
|
||||
"""结论里记的阈值:比率看最小期望频数,均值、逐行看最小样本数。"""
|
||||
return ["differ_alpha", "min_expected"] if mode == "rate" else ["differ_alpha", "min_n"]
|
||||
|
||||
|
||||
def refusal_basis(mode: str | None, settings: Mapping[str, Any]) -> tuple[list[str], str]:
|
||||
"""还没算就算不了时结论里记的阈值和判定标准(同 decompose.refusal_basis)。"""
|
||||
return used_settings(mode), ph.criterion_differ(settings, mode) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class Group:
|
||||
key: Any
|
||||
@@ -62,14 +76,21 @@ class Pair:
|
||||
|
||||
|
||||
def _int(value: Fraction | None, snap: Snap, r: int, column: str) -> int:
|
||||
if value is None or value.denominator != 1 or value < 0:
|
||||
if value is None:
|
||||
raise CallError("empty_cell", f"{snap.ref} 第 {r} 行的「{column}」是空的:计数要有值(在 SQL 里把空值写成 0)",
|
||||
reason=f"「{column}」那一格是空的({snap.ref} 第 {r} 行)")
|
||||
if value.denominator != 1 or value < 0:
|
||||
raise CallError("not_count", f"{snap.ref} 第 {r} 行的「{column}」不是不小于 0 的整数:比率比较的分子、分母要是计数")
|
||||
return int(value)
|
||||
|
||||
|
||||
#: 「其余各组合计」的键(不是分组列里的值)
|
||||
REST_KEY = ("__rest__",)
|
||||
|
||||
|
||||
def _pool(name: str, groups: Sequence[Group], mode: str) -> Group:
|
||||
"""「其余各组合计」:比率把成功数、试验数加起来;均值按各组的均值、方差、样本数合并;逐行的值并在一起。"""
|
||||
out = Group(("__rest__",), name, rows=[r for g in groups for r in g.rows])
|
||||
out = Group(REST_KEY, name, rows=[r for g in groups for r in g.rows])
|
||||
if mode == "rate":
|
||||
out.x, out.n = sum(g.x for g in groups), sum(g.n for g in groups)
|
||||
return out
|
||||
@@ -124,6 +145,22 @@ def _test(pair: Pair, mode: str, min_expected: Fraction) -> None:
|
||||
pair.ci = stats.welch_interval(float(a.mean), float(a.var), a.n, float(b.mean), float(b.var), b.n)
|
||||
|
||||
|
||||
def _gap(group: Group, mode: str) -> str | None:
|
||||
"""一组为什么算不了检验;算得了为 None。"""
|
||||
if mode == "rate":
|
||||
return ph.say(group.name, "的样本量为 0") if group.n == 0 else None
|
||||
if group.n < 2:
|
||||
return ph.say(group.name, "没有样本" if group.n == 0 else "只有 1 个样本")
|
||||
if group.mean is None or group.var is None:
|
||||
return ph.say(group.name, "的均值或标准差为空")
|
||||
return None
|
||||
|
||||
|
||||
def untested_text(pair: Pair, mode: str) -> str:
|
||||
"""这一对拿不到 p 值的原因:哪一组、怎么了(两组都有就都写)。"""
|
||||
return "、".join(dict.fromkeys(why for g in (pair.a, pair.b) if (why := _gap(g, mode)))) or "样本不够"
|
||||
|
||||
|
||||
def _verdict(pairs: Sequence[Pair], alpha: Fraction) -> list[Pair]:
|
||||
"""Holm 校正后站得住的那几对(按校正后的 p 从小到大,一样的按先后)。"""
|
||||
adjusted = stats.holm([p.p for p in pairs])
|
||||
@@ -135,7 +172,7 @@ def _verdict(pairs: Sequence[Pair], alpha: Fraction) -> list[Pair]:
|
||||
|
||||
def differ(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
mode = env.mode
|
||||
used = ["differ_alpha", "min_expected"] if mode == "rate" else ["differ_alpha", "min_n"]
|
||||
used = used_settings(mode)
|
||||
snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
|
||||
params: dict[str, Any] = {"query": snap.ref}
|
||||
by_period = raw.get("period") is not None
|
||||
@@ -230,7 +267,8 @@ def _groups(snap: Snap, group_col: str, cols: Mapping[str, str], mode: str, by_p
|
||||
if by_period:
|
||||
for w, what in ((params["target"], "本期"), (params["base"], "基期")):
|
||||
if not any(same_value(w, g.key) for g in groups.values()):
|
||||
raise CallError("period_not_found", f"期次列「{group_col}」里没有「{w}」:{what}要写 {snap.ref} 里有的期次值")
|
||||
raise CallError("period_not_found", f"期次列「{group_col}」里没有「{w}」:{what}要写 {snap.ref} 里有的期次值",
|
||||
reason=f"期次列「{group_col}」里没有{what}「{ph.key_text([w])}」")
|
||||
return groups
|
||||
|
||||
|
||||
@@ -238,7 +276,8 @@ def _find(groups: Mapping[Any, Group], value: Any, column: str) -> Group:
|
||||
found = next((g for g in groups.values() if same_value(value, g.key)), None)
|
||||
if found is None:
|
||||
shown = "、".join(g.name for g in list(groups.values())[:8])
|
||||
raise CallError("unknown_group", f"「{value}」不是「{column}」里的值:要比的组写 {shown} 里的")
|
||||
raise CallError("unknown_group", f"「{value}」不是「{column}」里的值:要比的组写 {shown} 里的",
|
||||
reason=f"分组列「{column}」里没有「{ph.key_text([value])}」这一组")
|
||||
return found
|
||||
|
||||
|
||||
@@ -261,7 +300,8 @@ def _pairs(raw: Mapping[str, Any], params: dict[str, Any], groups: dict[Any, Gro
|
||||
params["pairs"] = [[p.a.key, p.b.key] for p in out]
|
||||
return out
|
||||
if len(groups) < 2:
|
||||
raise CallError("too_few_groups", f"「{group_col}」里只有 {len(groups)} 组,至少要两组才比得了")
|
||||
raise CallError("too_few_groups", f"「{group_col}」里只有 {len(groups)} 组,至少要两组才比得了",
|
||||
reason=f"分组列「{group_col}」里只有 {len(groups)} 组,至少要两组才比得了")
|
||||
if raw.get("reference") is not None:
|
||||
ref = _find(groups, raw.get("reference"), group_col)
|
||||
params["reference"] = ref.key
|
||||
@@ -279,6 +319,12 @@ def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair]
|
||||
min_expected = env.threshold("min_expected") if mode == "rate" else Fraction(0)
|
||||
for pair in pairs:
|
||||
_test(pair, mode, min_expected)
|
||||
tested = [p for p in pairs if p.p is not None]
|
||||
untested = [p for p in pairs if p.p is None]
|
||||
if not tested:
|
||||
shown = list(dict.fromkeys(untested_text(p, mode) for p in pairs))
|
||||
return refuse(env, params, used, "untestable",
|
||||
",".join(shown[:3]) + ("等" if len(shown) > 3 else "") + ",算不了差异", criterion=criterion)
|
||||
hits = _verdict(pairs, alpha)
|
||||
caveats: list[dict[str, str]] = []
|
||||
if mode == "rows":
|
||||
@@ -293,24 +339,26 @@ def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair]
|
||||
small = sorted((g for p in pairs for g in (p.a, p.b) if g.n < min_n), key=lambda g: g.n)
|
||||
if small:
|
||||
caveats.append(ph.caveat("small_sample", ph.small_sample_mean_text(small[0].name, small[0].n, min_n)))
|
||||
tested = [p for p in pairs if p.p is not None]
|
||||
if len(tested) > 1:
|
||||
caveats.append(ph.caveat("multiple_comparisons", ph.multiple_text(len(tested))))
|
||||
if untested:
|
||||
caveats.append(ph.caveat("untested", ph.untested_pairs_text(len(untested),
|
||||
[untested_text(p, mode) for p in untested])))
|
||||
fmt = (lambda v: ph.pct(v, signed=False)) if mode == "rate" else (lambda v: ph.amount(v))
|
||||
if hits:
|
||||
code = "differs"
|
||||
head = hits[0]
|
||||
main = ph.differ_phrase([ph.differ_pair(p.a.name, p.b.name, label, fmt(p.a_value), fmt(p.b_value),
|
||||
p.diff > 0) for p in hits], # type: ignore[operator]
|
||||
len(pairs))
|
||||
len(tested))
|
||||
subjects = [[head.a.name], [head.b.name]]
|
||||
direction = sign_word(head.diff) # type: ignore[arg-type]
|
||||
else:
|
||||
code = "no_evidence"
|
||||
head = pairs[0] if len(pairs) == 1 else None
|
||||
head = tested[0] if len(tested) == 1 else None
|
||||
sizes = (ph.amount(head.a.n), ph.amount(head.b.n)) if head is not None else None
|
||||
main = ph.no_evidence_phrase(head.a.name if head else None, head.b.name if head else None, label, sizes,
|
||||
len(pairs))
|
||||
len(tested))
|
||||
subjects = [[head.a.name], [head.b.name]] if head is not None else []
|
||||
direction = None
|
||||
value_name = "比率" if mode == "rate" else "均值"
|
||||
@@ -318,7 +366,7 @@ def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair]
|
||||
for p in pairs:
|
||||
flags = []
|
||||
if p.p is None:
|
||||
flags.append("样本不够,算不了")
|
||||
flags.append(f"{untested_text(p, mode)},算不了")
|
||||
if p.test == "fisher":
|
||||
flags.append("样本少,用了精确检验")
|
||||
rows.append([p.a.name, p.b.name, canon(p.a_value), canon(p.b_value), canon(p.diff),
|
||||
@@ -326,7 +374,7 @@ def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair]
|
||||
TESTS.get(p.test or "") or None, canon(p.p), canon(p.p_adj), "、".join(flags) or None])
|
||||
columns = ["a", "b", "a_value", "b_value", "diff", "ci_lo", "ci_hi", "a_n", "b_n", "test", "p", "p_adj", "flag"]
|
||||
# 组、对照组照分组列(期次列)在快照里的类型;和「其余各组合计」比时对照组那一列有一格是叫法,写 text
|
||||
rest = any(p.b.key == ("__rest__",) for p in pairs)
|
||||
rest = any(p.b.key == REST_KEY for p in pairs)
|
||||
table = {"columns": columns,
|
||||
"labels": {"a": "组", "b": "对照组", "a_value": f"组的{value_name}", "b_value": f"对照组的{value_name}",
|
||||
"diff": "差", "ci_lo": "差的区间下限", "ci_hi": "差的区间上限", "a_n": "组的样本量",
|
||||
@@ -334,37 +382,32 @@ def _decide(env: Env, params: dict[str, Any], used: list[str], pairs: list[Pair]
|
||||
"rows": rows,
|
||||
"column_types": column_types(columns, dims={"a": kind, "b": "text" if rest else kind},
|
||||
texts=("test", "flag"))}
|
||||
shown = head if head is not None else pairs[0]
|
||||
shown = head if head is not None else tested[0]
|
||||
values = {"a": canon(shown.a_value), "b": canon(shown.b_value), "diff": canon(shown.diff),
|
||||
"ci_lo": canon(shown.ci[0]) if shown.ci else None, "ci_hi": canon(shown.ci[1]) if shown.ci else None,
|
||||
"p": canon(shown.p), "p_adj": canon(shown.p_adj), "n_a": str(shown.a.n), "n_b": str(shown.b.n),
|
||||
"test": shown.test, "pairs": str(len(pairs)), "differs": str(len(hits))}
|
||||
verdict = {"code": code, "subjects": subjects, "direction": direction}
|
||||
phrase = ph.with_caveats(main, caveats, skip=("multiple_comparisons",) if len(pairs) > 1 else ())
|
||||
phrase = ph.with_caveats(main, caveats, skip=("multiple_comparisons",) if len(tested) > 1 else ())
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion, phrase=phrase,
|
||||
caveats=caveats)
|
||||
|
||||
|
||||
def _sensitivity(pairs: list[Pair], mode: str, alpha: Fraction, min_expected: Fraction,
|
||||
differs: bool) -> tuple[bool, bool] | None:
|
||||
"""rows 模式的敏感性:去掉比较的各组里离均值最远的那一个值重算,判定变了交回 (去掉的是不是偏大的那一笔, 现在站不站得住)。"""
|
||||
everything = [(g, i, v) for g in dict.fromkeys(g for p in pairs for g in (p.a, p.b)) for i, v in enumerate(g.values)]
|
||||
"""rows 模式的敏感性:去掉比较的各组里离均值最远的那一个值重算,判定变了交回 (去掉的是不是偏大的那一笔, 现在站不站得住)。
|
||||
只在真实的各组里找(「其余各组合计」是它们的值并在一起,不另算一份),去掉以后由各组重建每一对的「其余」合计。"""
|
||||
groups = list(dict.fromkeys(g for p in pairs for g in (p.a, p.b) if g.key != REST_KEY))
|
||||
everything = [(g, i, v) for g in groups for i, v in enumerate(g.values)]
|
||||
if len(everything) < 3:
|
||||
return None
|
||||
mean = sum((v for _, _, v in everything), Fraction(0)) / len(everything)
|
||||
group, index, value = max(everything, key=lambda t: abs(t[2] - mean))
|
||||
trimmed = []
|
||||
for p in pairs:
|
||||
def cut(g: Group) -> Group:
|
||||
if g is not group and not (g.key == ("__rest__",) and any(v is value for v in g.values)):
|
||||
return g
|
||||
values = list(g.values)
|
||||
for k, v in enumerate(values):
|
||||
if v is value:
|
||||
del values[k]
|
||||
break
|
||||
return _moments(Group(g.key, g.name, rows=g.rows, values=values))
|
||||
trimmed.append(Pair(cut(p.a), cut(p.b)))
|
||||
cut = {g: g for g in groups}
|
||||
cut[group] = _moments(Group(group.key, group.name, rows=group.rows,
|
||||
values=[v for k, v in enumerate(group.values) if k != index]))
|
||||
trimmed = [Pair(cut[p.a], _pool(p.b.name, [cut[g] for g in groups if g is not p.a], mode)
|
||||
if p.b.key == REST_KEY else cut[p.b]) for p in pairs]
|
||||
for p in trimmed:
|
||||
_test(p, mode, min_expected)
|
||||
now = bool(_verdict(trimmed, alpha))
|
||||
@@ -372,4 +415,4 @@ def _sensitivity(pairs: list[Pair], mode: str, alpha: Fraction, min_expected: Fr
|
||||
return (value > mean, now) if now != differs else None
|
||||
|
||||
|
||||
__all__ = ["TESTS", "differ"]
|
||||
__all__ = ["REST_KEY", "TESTS", "differ", "refusal_basis", "untested_text", "used_settings"]
|
||||
|
||||
@@ -60,6 +60,7 @@ class Snap:
|
||||
dialect: str | None = None
|
||||
columns: list[str] = field(default_factory=list)
|
||||
rows: list[list[Any]] = field(default_factory=list)
|
||||
named: list[str] = field(default_factory=list) # 调用里认过的列(按先后,Inputs.settle 记用法用)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.columns = [str(c) for c in self.data.get("columns") or []]
|
||||
@@ -117,6 +118,8 @@ class Snap:
|
||||
column = hits[0]
|
||||
if column.lower() in self.masked:
|
||||
raise CallError("masked_column", f"「{column}」是遮罩列,不能当{role}:换一列,或者在 SQL 里不选它")
|
||||
if column not in self.named:
|
||||
self.named.append(column)
|
||||
return column
|
||||
|
||||
def index(self, column: str) -> int:
|
||||
@@ -157,18 +160,20 @@ class Inputs:
|
||||
self._resolve = resolve
|
||||
self._dialect = dialect
|
||||
self._snaps: dict[str, Snap] = {}
|
||||
self._roles: dict[str, str] = {}
|
||||
self.records: list[AnalysisInput] = []
|
||||
|
||||
def snap(self, name: Any, role: str) -> Snap:
|
||||
"""按名字取一份快照。"""
|
||||
def snap(self, name: Any, what: str, *, role: str = "data") -> Snap:
|
||||
"""按名字取一份快照。what 是调用错误里的叫法;role 是这份快照在结论里的角色(第一次取时的为准,settle 用)。"""
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise CallError("missing_query", f"没写{role}:写本节点查过的查询编号(如 Q3)")
|
||||
raise CallError("missing_query", f"没写{what}:写本节点查过的查询编号(如 Q3)")
|
||||
key = name.strip()
|
||||
if key not in self._snaps:
|
||||
found = self._resolve(key)
|
||||
if not isinstance(found, Mapping):
|
||||
raise CallError("unknown_query", f"没有叫「{key}」的查询结果:{role}写本节点查过的查询编号(如 Q3)")
|
||||
raise CallError("unknown_query", f"没有叫「{key}」的查询结果:{what}写本节点查过的查询编号(如 Q3)")
|
||||
self._snaps[key] = Snap(key, found, self._dialect(key, found) if self._dialect else found.get("dialect"))
|
||||
self._roles[key] = role
|
||||
return self._snaps[key]
|
||||
|
||||
def cell(self, ref: Any, role: str, label: str) -> tuple[Snap, int, str, Fraction | None]:
|
||||
@@ -176,10 +181,11 @@ class Inputs:
|
||||
found = _CELL.match(ref.strip()) if isinstance(ref, str) else None
|
||||
if found is None:
|
||||
raise CallError("bad_cell", f"{label}要写成单元格「编号.r行.列」(如 Q1.r0.gmv),现在是「{ref}」")
|
||||
snap = self.snap(found.group("name"), label)
|
||||
snap = self.snap(found.group("name"), label, role=role)
|
||||
row = int(found.group("row"))
|
||||
if row >= len(snap.rows):
|
||||
raise CallError("row_out_of_range", f"{snap.ref} 没有第 {row} 行(共 {len(snap.rows)} 行,从 r0 数起):改{label}")
|
||||
raise CallError("row_out_of_range", f"{snap.ref} 没有第 {row} 行(共 {len(snap.rows)} 行,从 r0 数起):改{label}",
|
||||
reason=f"{label}那一格({ref.strip()})不在:{snap.ref} 这次只有 {len(snap.rows)} 行")
|
||||
column = snap.column(found.group("column"), label)
|
||||
value = snap.number(row, column, label)
|
||||
self.record({"ref": f"{snap.ref}.r{row}.{column}", "role": role, "row": row, "column": column,
|
||||
@@ -209,6 +215,18 @@ class Inputs:
|
||||
if entry not in self.records:
|
||||
self.records.append(entry) # type: ignore[arg-type]
|
||||
|
||||
def settle(self) -> None:
|
||||
"""调用在算出结果之前就算不了时(随数据变化的调用错误记成算不了的结论):取过、还没记下用法的快照照样记一条(角色照
|
||||
第一次取时的,列是调用里认过的那几列,行记 all),复算才按名字取得回它们、走到同一处。"""
|
||||
recorded = set()
|
||||
for item in self.records:
|
||||
cell = _CELL.match(str(item.get("ref") or "")) if "row" in item else None
|
||||
recorded.add(cell.group("name") if cell else item.get("ref"))
|
||||
for key, snap in self._snaps.items():
|
||||
if key not in recorded:
|
||||
self.record({"ref": key, "role": self._roles.get(key, "data"), "columns": list(snap.named),
|
||||
"rows": "all"}, snap)
|
||||
|
||||
|
||||
def as_list(value: Any) -> list[Any]:
|
||||
"""参数里一个值或一组值:都规范成列表(None 是空列表)。"""
|
||||
|
||||
+325
-19
@@ -6,26 +6,32 @@
|
||||
且相邻两期隔整月 → month(月份都在 1、4、7、10 月且隔整季 → quarter);相邻两期都隔 7 天的整数倍 → week(这个日期当
|
||||
一周的第一天);否则 day。
|
||||
- 月份:'YYYY-MM'、'YYYYMM'(6 位,月份 01–12)→ month。
|
||||
- ISO 周:'YYYY-Www'(2026-W42 是 2026-10-12 到 10-18)→ week。SQLite 的 strftime('%W') 不是 ISO 周号(年初那几天
|
||||
算第 0 周),按它写的期次覆盖的日子会差一周:方法里要求期次写成每周第一天的日期。
|
||||
- 周号:'YYYY-Www' → week。同样写成 2026-W42,不同的数据库、不同的格式串算出来的是不同的日子(ISO 周是 10-12 到 10-18,
|
||||
SQLite 的 strftime('%Y-W%W') 是 10-19 到 10-25),所以起止按 SQL 里写这一列的格式串算(week_scheme 认出周号算法,见
|
||||
WEEK_SCHEMES);认不出算法的(方言不知道、只取了一列现成的字符串、年份和周号不配套)起止为空,本期齐不齐核对不了,先后按
|
||||
周号排(同一年里周号连着的算连着,上一年的第 52、53 周和下一年的第 0、1 周算连着)。按年截断的周号算法里,跨年那一周
|
||||
被切成两期,年末、年初那两期不到 7 天(short_days):不是整周,不能当本期、比较期,也不算进历史。
|
||||
- 季度:'YYYY-Qn'、'YYYYQn' → quarter。
|
||||
- 整数序号(不是上面几种的整数)→ seq:能排序、能判连续,不知道覆盖哪几天(本期完不完整核对不了)。
|
||||
|
||||
带时刻(非零点)的、写法混着的、认不出的都算认不出(period_unreadable)。
|
||||
|
||||
每一期记 ordinal:同一粒度里相邻两期的 ordinal 差 1(日:天数,周:从第一期起第几个 7 天,月:年 × 12 + 月,季:年 × 4 +
|
||||
季,序号:本身),据此排序、判缺期、按 lag 找比较期。start / end 是这一期覆盖的第一天和最后一天(含),seq 没有。
|
||||
每一期记 ordinal:同一粒度里相邻两期的 ordinal 差 1(日:天数,周:日期写的是从第一期起第几个 7 天、周号按它的算法排,
|
||||
月:年 × 12 + 月,季:年 × 4 + 季,序号:本身),据此排序、判缺期、按 lag 找比较期。start / end 是这一期覆盖的第一天和最后
|
||||
一天(含),seq 和认不出算法的周号没有。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import calendar
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
from app.data import coverage
|
||||
from app.data.names import name_key
|
||||
|
||||
GRAINS = ("day", "week", "month", "quarter")
|
||||
#: 粒度的叫法:数量词(过去 12 周、12 个月)、每…(每周、每月)
|
||||
@@ -38,6 +44,32 @@ _MONTH = re.compile(r"^(\d{4})-?(\d{2})$")
|
||||
_WEEK = re.compile(r"^(\d{4})-?W(\d{1,2})$", re.I)
|
||||
_QUARTER = re.compile(r"^(\d{4})-?Q([1-4])$", re.I)
|
||||
|
||||
#: 「YYYY-Www」的周号算法(week_scheme 按 SQL 认,week_span 按它算一周的起止):
|
||||
#: - iso:ISO 8601。周一开始,第 1 周是有 4 天以上落在这一年的那周,年份是 ISO 年(年初、年末的几天算到相邻的年)
|
||||
#: - monday:周一开始,第 1 周从这一年第一个周一起,之前的几天是第 0 周,按年截断(SQLite、C 的 %W)
|
||||
#: - sunday:周日开始,其余同 monday(%U;MySQL WEEK 模式 0)
|
||||
#: - monday4:周一开始,第 1 周是有 4 天以上落在这一年的那周,之前的几天是第 0 周,按年截断(MySQL 的 %u,WEEK 模式 1)
|
||||
#: - sunday_year:周日开始,第 1 周从这一年第一个周日起,年初之前的几天算上一年的最后一周(MySQL 的 %X 配 %V,WEEK 模式 2)
|
||||
#: - jan1:从 1 月 1 日起每 7 天一周,最后一周只有一两天(PostgreSQL、Oracle to_char 的 WW)
|
||||
WEEK_SCHEMES = ("iso", "monday", "sunday", "monday4", "sunday_year", "jan1")
|
||||
#: 各方言格式串里配套的(年份, 周号)写法 → 周号算法。不配套的(%Y 配 ISO 周号、IYYY 配 WW……)同一个标签会落在两段
|
||||
#: 日子上,认不出
|
||||
_WEEK_FORMATS: dict[str, dict[tuple[str, str], str]] = {
|
||||
"sqlite": {("%Y", "%W"): "monday", ("%Y", "%U"): "sunday", ("%G", "%V"): "iso"},
|
||||
"mysql": {("%Y", "%U"): "sunday", ("%Y", "%u"): "monday4", ("%X", "%V"): "sunday_year", ("%x", "%v"): "iso"},
|
||||
"postgres": {("YYYY", "WW"): "jan1", ("IYYY", "IW"): "iso"},
|
||||
"oracle": {("YYYY", "WW"): "jan1", ("IYYY", "IW"): "iso"},
|
||||
}
|
||||
#: 各方言格式串里年份、周号的写法(数格式串里有几个年份、几个周号用;方言不同,同一个 %u 意思不同)
|
||||
_YEAR_MARKS = {"sqlite": ("%Y", "%G"), "mysql": ("%Y", "%X", "%x"), "postgres": ("YYYY", "IYYY"),
|
||||
"oracle": ("YYYY", "IYYY")}
|
||||
_WEEK_MARKS = {"sqlite": ("%W", "%U", "%V"), "mysql": ("%U", "%u", "%V", "%v"), "postgres": ("WW", "IW"),
|
||||
"oracle": ("WW", "IW")}
|
||||
#: to_char 的格式串里先认长的:IYYY 里有 YYYY,IW、WW 都有 W(一个月里的第几周)
|
||||
_CHAR_MARKS = ("IYYY", "YYYY", "IW", "WW")
|
||||
#: 期次列从 CTE、子查询里原样取出时,最多往里找几层
|
||||
_NEST_MAX = 4
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Period:
|
||||
@@ -94,7 +126,7 @@ def _shape(value: Any) -> str | None:
|
||||
text = value.strip()
|
||||
if (m := _MONTH.match(text)) and 1 <= int(m.group(2)) <= 12 and 1900 <= int(m.group(1)) <= 2200:
|
||||
return "month"
|
||||
if _WEEK.match(text):
|
||||
if (m := _WEEK.match(text)) and int(m.group(2)) <= 53:
|
||||
return "week"
|
||||
if _QUARTER.match(text):
|
||||
return "quarter"
|
||||
@@ -111,6 +143,250 @@ def _month_of(value: Any) -> date:
|
||||
return date(int(m.group(1)), int(m.group(2)), 1)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 周号:按算法算起止、排先后
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _week_label(value: Any) -> tuple[int, int]:
|
||||
m = _WEEK.match(str(value).strip())
|
||||
assert m is not None
|
||||
return int(m.group(1)), int(m.group(2))
|
||||
|
||||
|
||||
def _first_weekday(start: date, weekday: int) -> date:
|
||||
"""start 当天或之后的第一个星期几(周一 0 … 周日 6)。"""
|
||||
return start + timedelta(days=(weekday - start.weekday()) % 7)
|
||||
|
||||
|
||||
def week_span(scheme: str, year: int, week: int) -> tuple[date, date] | None:
|
||||
"""按周号算法,year 年第 week 周覆盖的第一天和最后一天(含)。这种算法里没有这一周(ISO 只有 52 周的年份写了第 53 周、
|
||||
1 月 1 日就是周一的年份写了第 0 周……)、日子超出日期能表示的范围的为 None。"""
|
||||
try:
|
||||
return _week_span(scheme, year, week)
|
||||
except (OverflowError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _week_span(scheme: str, year: int, week: int) -> tuple[date, date] | None:
|
||||
if scheme == "iso":
|
||||
start = date.fromisocalendar(year, week, 1)
|
||||
return start, start + timedelta(days=6)
|
||||
jan1, dec31 = date(year, 1, 1), date(year, 12, 31)
|
||||
if scheme == "jan1":
|
||||
if not 1 <= week <= 53:
|
||||
return None
|
||||
start = jan1 + timedelta(days=7 * (week - 1))
|
||||
return start, min(start + timedelta(days=6), dec31)
|
||||
if scheme == "sunday_year":
|
||||
start = _first_weekday(jan1, 6) + timedelta(days=7 * (week - 1))
|
||||
return (start, start + timedelta(days=6)) if week >= 1 and start.year == year else None
|
||||
if scheme == "monday":
|
||||
week1 = _first_weekday(jan1, 0)
|
||||
elif scheme == "sunday":
|
||||
week1 = _first_weekday(jan1, 6)
|
||||
elif scheme == "monday4":
|
||||
week1 = date.fromisocalendar(year, 1, 1) # ISO 第 1 周的周一,可能在上一年的 12 月
|
||||
else:
|
||||
return None
|
||||
if week == 0: # 第 1 周之前的那几天;第 1 周从 1 月 1 日(或更早)起就没有
|
||||
return (jan1, week1 - timedelta(days=1)) if week1 > jan1 else None
|
||||
start = week1 + timedelta(days=7 * (week - 1))
|
||||
if week < 0 or start > dec31:
|
||||
return None
|
||||
return max(start, jan1), min(start + timedelta(days=6), dec31)
|
||||
|
||||
|
||||
def _week_start(day: date, weekday: int) -> date:
|
||||
"""day 所在那一周(从星期几开始)的第一天。"""
|
||||
return day - timedelta(days=(day.weekday() - weekday) % 7)
|
||||
|
||||
|
||||
def _week_ordinal(scheme: str, span: tuple[date, date], year: int, base: int) -> int:
|
||||
"""一周在这种算法的先后里排第几(相邻两期差 1)。整 7 天一周的按那一周第一天的序数除以 7(同一个星期几开头的日子序数
|
||||
模 7 同余);按年截断的,跨年那一周切成了两期,每过一个不是从这周第一天开始的元旦多一期(base 是这一列里最早的年份)。"""
|
||||
if scheme == "jan1":
|
||||
return year * 53 + (span[0] - date(year, 1, 1)).days // 7
|
||||
if scheme in ("iso", "sunday_year"):
|
||||
return span[0].toordinal() // 7
|
||||
weekday = 6 if scheme == "sunday" else 0
|
||||
splits = sum(1 for y in range(base + 1, year + 1) if date(y, 1, 1).weekday() != weekday)
|
||||
return _week_start(span[0], weekday).toordinal() // 7 + splits
|
||||
|
||||
|
||||
def _loose_ordinals(labels: Iterable[tuple[int, int]]) -> dict[tuple[int, int], int]:
|
||||
"""认不出周号算法时的先后:按(年,周号)排,同一年里周号连着的算连着;上一年的第 52、53 周和下一年的第 0、1 周算连着
|
||||
(核对不了跨年那里是不是缺了一周);别的跨年当缺期(隔几期只是估计,至少隔一期)。"""
|
||||
out: dict[tuple[int, int], int] = {}
|
||||
prev: tuple[int, int] | None = None
|
||||
n = 0
|
||||
for year, week in sorted(set(labels)):
|
||||
if prev is not None:
|
||||
if year == prev[0]:
|
||||
n += week - prev[1]
|
||||
elif year == prev[0] + 1 and prev[1] >= 52 and week <= 1:
|
||||
n += 1
|
||||
else:
|
||||
n += max(2, (year - prev[0]) * 53 + week - prev[1])
|
||||
out[(year, week)] = n
|
||||
prev = (year, week)
|
||||
return out
|
||||
|
||||
|
||||
def short_days(item: Period, grain: str) -> int | None:
|
||||
"""这一期不到整期(按年截断的周号算法里跨年切开的那两期、WW 的最后一周)时有几天;整期、不知道起止的为 None。"""
|
||||
if grain != "week" or item.start is None or item.end is None:
|
||||
return None
|
||||
days = (item.end - item.start).days + 1
|
||||
return days if days < 7 else None
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def week_scheme(sql: str, column: str, dialect: str | None) -> str | None:
|
||||
"""期次列 column 是按哪种周号算法写成「YYYY-Www」的(WEEK_SCHEMES 之一):看最外层 SELECT 里这一列怎么写(从 CTE、子查询
|
||||
里原样取出的往里找),格式化函数(SQLite 的 strftime、MySQL 的 DATE_FORMAT、PostgreSQL / Oracle 的 to_char)的格式串里
|
||||
年份和周号是不是配套的一对(_WEEK_FORMATS),而且只引用了一列。认不出(方言不知道、不是这几个函数、年份和周号不配套或
|
||||
不止一个、只是取了一列现成的字符串)为 None。"""
|
||||
from sqlglot import exp
|
||||
|
||||
pairs = _WEEK_FORMATS.get(dialect or "")
|
||||
if pairs is None or not sql:
|
||||
return None
|
||||
try:
|
||||
value = _period_expr(sql, column, dialect)
|
||||
if value is None:
|
||||
return None
|
||||
refs = {(name_key(c.table), name_key(c.name)) for c in value.find_all(exp.Column)
|
||||
if not isinstance(c.this, exp.Star)}
|
||||
texts = [_native_format(node, dialect) # type: ignore[arg-type]
|
||||
for node in value.find_all(exp.TimeToStr, exp.ToChar)]
|
||||
if len(refs) != 1 or not texts or any(t is None for t in texts):
|
||||
return None
|
||||
marks = [m for t in texts for m in _format_marks(t, dialect)] # type: ignore[arg-type]
|
||||
years = [m for m in marks if m in _YEAR_MARKS[dialect]] # type: ignore[index]
|
||||
weeks = [m for m in marks if m in _WEEK_MARKS[dialect]] # type: ignore[index]
|
||||
if len(years) != 1 or len(weeks) != 1:
|
||||
return None
|
||||
return pairs.get((years[0], weeks[0]))
|
||||
except Exception: # noqa: BLE001 - 认不出就当不知道算法
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def plain_day(sql: str, column: str, dialect: str | None) -> bool:
|
||||
"""期次列是不是直接取的一个日期列(原样、date(列)、DATE(列)、CAST(列 AS DATE)),没有按周、按月换算过。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.engine import cell_format as cf
|
||||
|
||||
try:
|
||||
value = _period_expr(sql, column, dialect)
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
if isinstance(value, (exp.Date, exp.TsOrDsToDate)):
|
||||
extra = [v for k, v in value.args.items() if k != "this" and v not in (None, [], False)]
|
||||
value = cf._unwrap(value.this) if not extra else None
|
||||
return isinstance(value, exp.Column) and not isinstance(value.this, exp.Star)
|
||||
|
||||
|
||||
def parse_compared(values: Sequence[Any], sql: str, column: str,
|
||||
dialect: str | None) -> tuple[list[Period], str] | None:
|
||||
"""按期次分组的一份快照里比的两期(变化拆解、可比性检查没给粒度时;快照里往往只有这两期):同 parse,周号按 SQL 认算法,
|
||||
粒度按值推断;但值都是日期、推断成按周、期次列又是直接取的日期列(plain_day)时按天认——两期正好隔 7 天,光看值分不出
|
||||
是两天还是两周,期次列没按周换算过就是那一天。"""
|
||||
parsed = parse(values, weeks=week_scheme(sql, column, dialect))
|
||||
if parsed is not None and parsed[1] == "week" and all(_shape(v) == "date" for v in values) \
|
||||
and plain_day(sql, column, dialect):
|
||||
return parse(values, "day")
|
||||
return parsed
|
||||
|
||||
|
||||
def _period_expr(sql: str, column: str, dialect: str | None) -> Any:
|
||||
"""结果列 column 的写法(sqlglot 节点):最外层 SELECT 里那一项;是从 CTE、子查询里原样取出的一列时往里找。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.engine import cell_format as cf
|
||||
|
||||
tree = coverage.parse_sql(sql, dialect)
|
||||
if not isinstance(tree, exp.Select):
|
||||
return None
|
||||
ctes = {name_key(c.alias_or_name): c.this for c in tree.find_all(exp.CTE)}
|
||||
select, name = tree, column
|
||||
for _ in range(_NEST_MAX + 1):
|
||||
item = cf._projection(select, name)
|
||||
if item is None:
|
||||
return None
|
||||
value = cf._unwrap(item.unalias())
|
||||
if not isinstance(value, exp.Column) or isinstance(value.this, exp.Star):
|
||||
return value
|
||||
inner = _inner_select(select, value, ctes)
|
||||
if inner is None:
|
||||
return value
|
||||
select, name = inner, value.name
|
||||
return None
|
||||
|
||||
|
||||
def _inner_select(select: Any, column: Any, ctes: dict[str, Any]) -> Any:
|
||||
"""外层取的这一列来自哪个 CTE、子查询(那一层的 SELECT);来自实表、认不准是哪个来源的为 None。"""
|
||||
from sqlglot import exp
|
||||
|
||||
sources = [select.args.get("from_") or select.args.get("from"), *(select.args.get("joins") or [])]
|
||||
found = [s.this for s in sources if s is not None and getattr(s, "this", None) is not None]
|
||||
if column.table:
|
||||
found = [n for n in found if name_key(n.alias_or_name) == name_key(column.table)]
|
||||
if len(found) != 1:
|
||||
return None
|
||||
node = found[0]
|
||||
if isinstance(node, exp.Subquery):
|
||||
inner = node.this
|
||||
elif isinstance(node, exp.Table) and not node.db and name_key(node.name) in ctes:
|
||||
inner = ctes[name_key(node.name)]
|
||||
else:
|
||||
return None
|
||||
return inner if isinstance(inner, exp.Select) else None
|
||||
|
||||
|
||||
def _native_format(node: Any, dialect: str) -> str | None:
|
||||
"""格式化函数的格式串,按这个方言原本的写法(sqlglot 解析时换成了自己的写法,再换回去;Oracle 的 to_char 原样留着)。"""
|
||||
from sqlglot import exp
|
||||
from sqlglot.dialects.dialect import Dialect
|
||||
|
||||
fmt = node.args.get("format")
|
||||
if not isinstance(fmt, exp.Literal) or not fmt.is_string:
|
||||
return None
|
||||
if isinstance(node, exp.ToChar):
|
||||
return str(fmt.this)
|
||||
text = Dialect.get_or_raise(dialect).generator().format_time(node)
|
||||
if not text or len(text) < 2 or text[0] != "'" or text[-1] != "'":
|
||||
return None
|
||||
return text[1:-1].replace("''", "'")
|
||||
|
||||
|
||||
def _format_marks(text: str, dialect: str) -> list[str]:
|
||||
"""格式串里的格式符(按先后):strftime、DATE_FORMAT 是「%」加一个字(%% 是百分号本身);to_char 去掉双引号里的原样文字后
|
||||
按 _CHAR_MARKS 从左往右认(不分大小写)。"""
|
||||
out: list[str] = []
|
||||
i = 0
|
||||
if dialect in ("sqlite", "mysql"):
|
||||
while i < len(text):
|
||||
if text[i] == "%" and i + 1 < len(text):
|
||||
if text[i + 1] != "%":
|
||||
out.append(text[i:i + 2])
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
return out
|
||||
plain = re.sub(r'"[^"]*"', "", text).upper()
|
||||
while i < len(plain):
|
||||
mark = next((m for m in _CHAR_MARKS if plain.startswith(m, i)), None)
|
||||
if mark is not None:
|
||||
out.append(mark)
|
||||
i += len(mark)
|
||||
else:
|
||||
i += 1
|
||||
return out
|
||||
|
||||
|
||||
def _infer_date_grain(days: Sequence[date]) -> str:
|
||||
"""一列日期的粒度(见文件头)。"""
|
||||
ordered = sorted(set(days))
|
||||
@@ -131,8 +407,10 @@ def _infer_date_grain(days: Sequence[date]) -> str:
|
||||
return "day"
|
||||
|
||||
|
||||
def parse(values: Sequence[Any], grain: str | None = None) -> tuple[list[Period], str] | None:
|
||||
"""一列期次值 → (逐个的 Period,粒度)。认不出(写法不一、带时刻、粒度和写法对不上)返回 None。"""
|
||||
def parse(values: Sequence[Any], grain: str | None = None, *,
|
||||
weeks: str | None = None) -> tuple[list[Period], str] | None:
|
||||
"""一列期次值 → (逐个的 Period,粒度)。认不出(写法不一、带时刻、粒度和写法对不上、周号在 weeks 这种算法里没有)返回 None,
|
||||
是哪几个值见 unreadable。weeks 是 week_scheme 认出的周号算法:只用于「YYYY-Www」,None 是认不出算法(起止为空)。"""
|
||||
shapes = {_shape(v) for v in values}
|
||||
if not values or None in shapes or len(shapes) != 1:
|
||||
return None
|
||||
@@ -152,16 +430,16 @@ def parse(values: Sequence[Any], grain: str | None = None) -> tuple[list[Period]
|
||||
if shape == "week":
|
||||
if grain not in (None, "week"):
|
||||
return None
|
||||
out = []
|
||||
for v in values:
|
||||
m = _WEEK.match(str(v).strip())
|
||||
assert m is not None
|
||||
try:
|
||||
start = date.fromisocalendar(int(m.group(1)), int(m.group(2)), 1)
|
||||
except ValueError:
|
||||
return None
|
||||
out.append(Period(v, start.toordinal() // 7, start, start + timedelta(days=6)))
|
||||
return out, "week"
|
||||
labels = [_week_label(v) for v in values]
|
||||
if weeks is None:
|
||||
order = _loose_ordinals(labels)
|
||||
return [Period(v, order[label], None, None) for v, label in zip(values, labels)], "week"
|
||||
spans = [week_span(weeks, *label) for label in labels]
|
||||
if any(s is None for s in spans):
|
||||
return None
|
||||
base = min(year for year, _ in labels)
|
||||
return [Period(v, _week_ordinal(weeks, s, label[0], base), s[0], s[1]) # type: ignore[index, arg-type]
|
||||
for v, label, s in zip(values, labels, spans)], "week"
|
||||
if shape == "quarter":
|
||||
if grain not in (None, "quarter"):
|
||||
return None
|
||||
@@ -196,6 +474,33 @@ def parse(values: Sequence[Any], grain: str | None = None) -> tuple[list[Period]
|
||||
return None
|
||||
|
||||
|
||||
def unreadable(values: Sequence[Any], grain: str | None = None, *,
|
||||
weeks: str | None = None) -> tuple[str, list[Any]]:
|
||||
"""parse 认不出时,为什么、是哪几个值(按先后、去重):shape 写法认不出(空值、带时刻、不是这几种写法的);mixed 写法混着
|
||||
(不是最多那种写法的几个,一样多时以先出现的为准);week 周号在 SQL 的算法里没有这一周;grain 和给的粒度对不上(全部)。"""
|
||||
shapes = [_shape(v) for v in values]
|
||||
odd = [v for v, s in zip(values, shapes) if s is None]
|
||||
kinds = list(dict.fromkeys(shapes))
|
||||
if odd or not values:
|
||||
return "shape", _distinct(odd)
|
||||
if len(kinds) > 1:
|
||||
main = max(kinds, key=shapes.count)
|
||||
return "mixed", _distinct(v for v, s in zip(values, shapes) if s != main)
|
||||
if kinds[0] == "week" and weeks is not None and grain in (None, "week"):
|
||||
odd = [v for v in values if week_span(weeks, *_week_label(v)) is None]
|
||||
if odd:
|
||||
return "week", _distinct(odd)
|
||||
return "grain", _distinct(values)
|
||||
|
||||
|
||||
def _distinct(values: Iterable[Any]) -> list[Any]:
|
||||
out: list[Any] = []
|
||||
for v in values:
|
||||
if not any(v == o and type(v) is type(o) for o in out):
|
||||
out.append(v)
|
||||
return out
|
||||
|
||||
|
||||
def calendar_unit(start: date, end: date) -> str | None:
|
||||
"""这段日子是不是一个整的日历期:整月 month、整季 quarter、整年 year;都不是为 None。整月对整月算一样长(31 天的 10 月
|
||||
和 30 天的 9 月环比是通行的比法),不按天数判窗口不等。"""
|
||||
@@ -221,4 +526,5 @@ def weekend_days(start: date, end: date) -> int:
|
||||
return count
|
||||
|
||||
|
||||
__all__ = ["COUNT_WORD", "DEFAULT_LAG", "GRAINS", "PER_WORD", "Period", "calendar_unit", "parse", "weekend_days"]
|
||||
__all__ = ["COUNT_WORD", "DEFAULT_LAG", "GRAINS", "PER_WORD", "WEEK_SCHEMES", "Period", "calendar_unit", "parse",
|
||||
"parse_compared", "plain_day", "short_days", "unreadable", "week_scheme", "week_span", "weekend_days"]
|
||||
|
||||
@@ -11,9 +11,11 @@
|
||||
- 百分点:比率的变化(2.1% 到 2.5% 是上升 0.4 个百分点)。
|
||||
|
||||
**label 消毒**(§7.3):判定语里的度量叫法、因子名、维度叫法是 agent 或数据给的。agent 给的 label(以及目录里的叫法、
|
||||
当叫法用的列名)去掉数字、百分号、单位符号、引用标记,不超过 20 字,消毒后为空就退回下一个来源:不然模型可以把
|
||||
「销售额(同比 +30%)」塞进 label,借系统渲染绕过裸数字检测。去数字按出具校验抽数字的同一个函数(engine/issuance 的
|
||||
extract_numbers:阿拉伯数字、中文数字、带万 / 亿 / 成 / 倍的写法),再去掉剩下的数字字符。维度值是数据,照原样。
|
||||
当叫法用的列名)去掉数字、百分号、单位符号、引用标记,截到第一个分句标点之前,不超过 20 字;里面有说法表的判断词的整个
|
||||
不要;消毒后为空就退回下一个来源:不然模型可以把「销售额(同比 +30%)」「销售额,主要来自华东」塞进 label,借系统渲染绕过
|
||||
裸数字检测和说法核对(判定语不受说法核对)。去数字按出具校验抽数字的同一个函数(engine/issuance 的 extract_numbers:阿拉伯
|
||||
数字、中文数字、带万 / 亿 / 成 / 倍的写法),再去掉剩下的数字字符。消毒反复做到不再变化:消过的再消一遍不变,复算拿记下的
|
||||
叫法再消,判定语逐字相同。维度值是数据,照原样。
|
||||
|
||||
**附带说明**:判定语后面用「;」接,最多两条(按 PHRASE_ORDER 的先后),其余只在证据面板。度量类型没标、以各段之和为
|
||||
总数这类方法上的交代不进判定语。
|
||||
@@ -29,9 +31,9 @@ from typing import Any
|
||||
from app.analysis.contract import OP_LABELS
|
||||
from app.analysis.numbers import fixed, to_fraction
|
||||
|
||||
#: 附带说明进判定语的先后;不在这里的(measure_unlabeled、headline_basis)只在证据面板
|
||||
PHRASE_ORDER = ("comparability", "simpson", "offset", "outlier_sensitive", "small_sample", "record_high", "record_low",
|
||||
"multiple_comparisons", "weekday_effect", "small_denominator", "null_values")
|
||||
#: 附带说明进判定语的先后;不在这里的(measure_unlabeled、headline_basis、partial_periods)只在证据面板
|
||||
PHRASE_ORDER = ("comparability", "untested", "simpson", "offset", "outlier_sensitive", "small_sample", "record_high",
|
||||
"record_low", "multiple_comparisons", "weekday_effect", "small_denominator", "null_values")
|
||||
#: 判定语里最多接几条附带说明
|
||||
PHRASE_CAVEATS = 2
|
||||
#: 消毒后的叫法最多几个字
|
||||
@@ -168,13 +170,63 @@ _DIGITS = re.compile(r"[0-90-9]")
|
||||
_SYMBOLS = re.compile(r"[%%‰‱¥¥$$€£₩℃°++\-−–±×÷==<><>~~^*]")
|
||||
_EMPTY_BRACKETS = re.compile(r"[((【\[]\s*[))】\]]")
|
||||
_EDGE_PUNCT = ",,、::;;。.!!?? "
|
||||
#: 分句的标点:叫法截到第一个之前(「销售额,主要来自华东」只留「销售额」)
|
||||
_CLAUSE = re.compile(r"[,。;:!?、,.;:!?]")
|
||||
#: 截出来以后还没配上对的左括号(「销售额(含税,元)」截成「销售额(含税」)连同后面的都不要
|
||||
_OPENERS = {"(": ")", "(": ")", "【": "】"}
|
||||
|
||||
|
||||
def clean_label(text: Any) -> str:
|
||||
"""消毒(见文件头):去引用标记、数字(出具校验认的数字写法连同后缀,再加剩下的数字字符)、百分号和单位符号、空括号,
|
||||
压掉多余空白,不超过 LABEL_MAX 字。可能消成空串。"""
|
||||
"""消毒(见文件头):去引用标记、数字(出具校验认的数字写法连同后缀,再加剩下的数字字符)、百分号和单位符号、空括号、
|
||||
多余空白,反复做到不再变化(去掉一处可能拼出新的一处:「销售额[ [v:Q1.r0.gmv] ]」去完数字成了「销售额[[v:Q.r.gmv]]」),
|
||||
再去掉剩下的方括号,截到第一个分句标点之前、去掉没配上对的左括号,不超过 LABEL_MAX 字;整个过程再做到不再变化,所以
|
||||
消过的再消一遍不变。消完的叫法里有说法表的判断词(engine/analysis_claims.scan 认得出的)就整个不要。可能消成空串。"""
|
||||
if not isinstance(text, str):
|
||||
return ""
|
||||
out = text
|
||||
for _ in range(len(text) + 2):
|
||||
again = _shorten(_settle(out))
|
||||
if again == out:
|
||||
break
|
||||
out = again
|
||||
return "" if out and _judges(out) else out
|
||||
|
||||
|
||||
def _settle(text: str) -> str:
|
||||
"""去引用标记、数字、符号、空括号,做到不再变化(每一遍只会变短或只换空白,一定停得下来)。"""
|
||||
out = text
|
||||
for _ in range(len(text) + 2):
|
||||
again = _scrub(out)
|
||||
if again == out:
|
||||
break
|
||||
out = again
|
||||
return out
|
||||
|
||||
|
||||
def _shorten(text: str) -> str:
|
||||
"""去掉剩下的方括号,截到第一个分句标点之前,去掉没配上对的左括号连同后面的,不超过 LABEL_MAX 字。"""
|
||||
out = _CLAUSE.split(text.replace("[", "").replace("]", ""), maxsplit=1)[0]
|
||||
open_at: list[tuple[int, str]] = []
|
||||
for i, ch in enumerate(out):
|
||||
if ch in _OPENERS:
|
||||
open_at.append((i, _OPENERS[ch]))
|
||||
elif open_at and ch == open_at[-1][1]:
|
||||
open_at.pop()
|
||||
if open_at:
|
||||
out = out[:open_at[0][0]]
|
||||
out = re.sub(r"\s+", " ", out).strip(_EDGE_PUNCT)
|
||||
return out[:LABEL_MAX].strip(_EDGE_PUNCT)
|
||||
|
||||
|
||||
def _judges(text: str) -> bool:
|
||||
"""叫法里有没有说法表的判断词(「主要来自」「历史新高」……)。说法核对在 engine 里,在这里导入免得循环导入。"""
|
||||
from app.engine.analysis_claims import scan
|
||||
|
||||
return bool(scan(text))
|
||||
|
||||
|
||||
def _scrub(text: str) -> str:
|
||||
"""一遍:去引用标记、数字、符号,收拾去掉以后留下的空白、标点、空括号。"""
|
||||
from app.engine.issuance import extract_numbers
|
||||
|
||||
raw = _MARKER.sub("", text)
|
||||
@@ -193,8 +245,7 @@ def clean_label(text: Any) -> str:
|
||||
out = re.sub(r"([,,、::;;])[,,、::;;]+", r"\1", out)
|
||||
while _EMPTY_BRACKETS.search(out):
|
||||
out = _EMPTY_BRACKETS.sub("", out)
|
||||
out = re.sub(r"\s+", " ", out).strip(_EDGE_PUNCT)
|
||||
return out[:LABEL_MAX].strip(_EDGE_PUNCT)
|
||||
return re.sub(r"\s+", " ", out).strip(_EDGE_PUNCT)
|
||||
|
||||
|
||||
def pick_label(*candidates: Any, fallback: str = "这一列") -> str:
|
||||
@@ -250,13 +301,16 @@ def offset_text(delta_total: Any, gross_up: Any, gross_down: Any) -> str:
|
||||
return f"下降的段合计 {down},上升的段合计 {up},互相抵消了一部分"
|
||||
|
||||
|
||||
def simpson_text(label: str, overall_up: bool, subject: str, w_base: Any, w_target: Any) -> str:
|
||||
"""辛普森:各段的比率大多和整体反着变,整体的变化来自结构。"""
|
||||
def simpson_text(label: str, overall_up: bool, subject: str, w_base: Any, w_target: Any, *, shift: bool = True) -> str:
|
||||
"""辛普森:各段的比率大多和整体反着变,整体的变化来自结构。判定语正文已经说了结构变化、点了那一段(mix_driven)时
|
||||
shift 为假,只说各段和整体反着变,结构变化不说第二遍。"""
|
||||
seg = "上升" if not overall_up else "下降"
|
||||
whole = "上升" if overall_up else "下降"
|
||||
head = say("各段的", label, f"大多在{seg},整体却{whole}")
|
||||
if not shift:
|
||||
return head
|
||||
move = "升" if _q(w_target) > _q(w_base) else "降"
|
||||
return say(f"各段的", label, f"大多在{seg},整体却{whole}:变化来自结构,", subject,
|
||||
f"占比从 {pct(w_base, signed=False)} {move}到 {pct(w_target, signed=False)}")
|
||||
return say(head, ":变化来自结构,", subject, f"占比从 {pct(w_base, signed=False)} {move}到 {pct(w_target, signed=False)}")
|
||||
|
||||
|
||||
def small_denominator_text(segments: Sequence[str], denominator: str, minimum: int) -> str:
|
||||
@@ -292,6 +346,12 @@ def multiple_text(pairs: int) -> str:
|
||||
return f"比了 {pairs} 对,已按次数校正"
|
||||
|
||||
|
||||
def untested_pairs_text(count: int, reasons: Sequence[str]) -> str:
|
||||
"""差异检验里拿不到 p 值的那几对(不算进比了几对):另有 1 对算不了(华北的样本量为 0)。"""
|
||||
shown = list(dict.fromkeys(reasons))
|
||||
return f"另有 {count} 对算不了({','.join(shown[:3])}{'等' if len(shown) > 3 else ''})"
|
||||
|
||||
|
||||
def outlier_text(largest: bool, now_differs: bool) -> str:
|
||||
which = "最大" if largest else "最小"
|
||||
return f"去掉{which}的一笔后,差异{'变得站得住' if now_differs else '不再站得住'}"
|
||||
@@ -306,6 +366,13 @@ def small_sample_mean_text(group: str, n: int, minimum: int) -> str:
|
||||
|
||||
|
||||
UNVERIFIED_COMPLETE = "没能核对本期的数据是否到齐(查不到表里这一列的日期范围)"
|
||||
UNKNOWN_WEEKS = "没能核对本期的数据是否到齐(期次写成「年-W周号」,认不出 SQL 是按哪种周号算的,不知道是哪几天)"
|
||||
NO_DATES = "没能核对本期的数据是否到齐(期次不是日期,不知道是哪几天)"
|
||||
|
||||
|
||||
def partial_periods_text(count: int) -> str:
|
||||
"""按年截断的周号算法里跨年切开的那几期(不到整周)没有算进历史。"""
|
||||
return f"有 {count} 期不到整周(跨年被切开),没有算进历史"
|
||||
|
||||
|
||||
def with_caveats(main: str, caveats: Sequence[Mapping[str, str]], *, skip: Iterable[str] = ()) -> str:
|
||||
@@ -414,13 +481,15 @@ def baseline_flat_phrase(verdict: str, label: str, current: str, n: int, unit: s
|
||||
|
||||
def trend_phrase(verdict: str, label: str, n: int, unit: str, per: str, slope: str | None,
|
||||
run: tuple[int, str] | None) -> str:
|
||||
"""趋势。末尾连续同向的期数(run)和趋势同向时接「,已连续 3 周下降」,反向时接「,但最近已连续 3 周上升」。"""
|
||||
if verdict == "no_trend":
|
||||
return say(f"过去 {n} {unit}", label, "没有明显的上升或下降趋势")
|
||||
word = "上升" if verdict == "rising" else "下降"
|
||||
text = say(f"过去 {n} {unit}", label, f"呈{word}趋势") + (f"(每{per}约 {slope})" if slope else "")
|
||||
if run is not None:
|
||||
length, direction = run
|
||||
text += f",已连续 {length} {unit}{'上升' if direction == 'up' else '下降'}"
|
||||
same = (direction == "up") == (verdict == "rising")
|
||||
text += f",{'' if same else '但最近'}已连续 {length} {unit}{'上升' if direction == 'up' else '下降'}"
|
||||
return text
|
||||
|
||||
|
||||
@@ -501,14 +570,16 @@ CRITERION_COMPARABLE = ("只差日期、窗口一样长、是整期、星期构
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CRITERION_COMPARABLE", "EMPTY_KEY", "HEADLINE_BASIS", "KEY_JOIN", "LABEL_MAX", "MEASURE_WORDS",
|
||||
"PHRASE_CAVEATS", "PHRASE_ORDER", "UNVERIFIED_COMPLETE", "WEEKDAY_EFFECT", "amount", "band_text",
|
||||
"CRITERION_COMPARABLE", "EMPTY_KEY", "HEADLINE_BASIS", "KEY_JOIN", "LABEL_MAX", "MEASURE_WORDS", "NO_DATES",
|
||||
"PHRASE_CAVEATS", "PHRASE_ORDER", "UNKNOWN_WEEKS", "UNVERIFIED_COMPLETE", "WEEKDAY_EFFECT", "amount", "band_text",
|
||||
"baseline_flat_phrase", "baseline_phrase", "caveat", "change", "clean_label", "comparable_phrase",
|
||||
"criterion_additive", "criterion_baseline", "criterion_differ", "criterion_factor", "criterion_ratio",
|
||||
"criterion_trend", "current_text", "decompose_concentrated", "decompose_dispersed", "decompose_driver",
|
||||
"decompose_flat", "decompose_unclosed", "differ_pair", "differ_phrase", "dims_text", "factor_both",
|
||||
"factor_driver", "key_text", "multiple_text", "names", "no_evidence_phrase", "null_periods_text",
|
||||
"null_segments_text", "number", "offset_text", "outlier_text", "pct", "pick_label", "points", "ratio_both",
|
||||
"null_segments_text", "number", "offset_text", "outlier_text", "partial_periods_text", "pct", "pick_label",
|
||||
"untested_pairs_text",
|
||||
"points", "ratio_both",
|
||||
"ratio_change", "ratio_mix", "ratio_rate", "ratio_value", "record_text", "refused", "say", "share_words",
|
||||
"simpson_text", "small_denominator_text", "small_sample_mean_text", "small_sample_rate_text", "threshold_pct",
|
||||
"trend_phrase", "unconfirmed_text", "unlabeled_text", "with_caveats",
|
||||
|
||||
+107
-36
@@ -5,9 +5,11 @@ week / month / quarter,缺省按期次值推断,见 analysis/periods)、la
|
||||
(change 缺省:和比较期比的变化率;level:值本身,给存量、比率用)、lag(和往前第几期比,缺省日数据 7、其余 1;同比写 52 /
|
||||
12 / 4,历史要相应够长)。
|
||||
|
||||
**前提**(不满足就算不了,原因照实写):期次认得出(period_unreadable)、本期完整(同可比性检查的整期:没过完、数据没到、
|
||||
SQL 截掉一部分 → partial_target;表的日期范围查不到时只按运行当天判,附一句核对不了)、本期和比较期之间不缺期(target_gap)、
|
||||
比较期不为 0(zero_base)、历史够长(history_short);趋势要最后几期连着(中间缺期 series_gap)。度量格为空的那一期跳过。
|
||||
**前提**(不满足就算不了,原因照实写):期次认得出(period_unreadable,原因里举认不出的那几个值)、本期完整(同可比性检查的
|
||||
整期:不到整周、没过完、数据没到、SQL 截掉一部分 → partial_target;表的日期范围查不到时只按运行当天判,附一句核对不了;期次
|
||||
没有日期、周号认不出算法的不判齐,附一句核对不了)、比较期有值(缺期、那一期的值为空、不到整周 → target_gap,原因照实分开
|
||||
写)、比较期不为 0(zero_base)、历史够长(history_short);趋势要最后几期连着(中间缺期、值为空、不到整周 series_gap)。度量
|
||||
格为空的那一期跳过;不到整周的那几期(按年截断的周号算法里跨年切开的)不算进历史。
|
||||
|
||||
**波动基线**:历史的中位数 m、中位绝对偏差 MAD,σ̂ = 1.4826·MAD,稳健 z = (x − m)/σ̂;常规波动范围 m ± k_warn·σ̂;全程
|
||||
精确的有理数。σ̂ = 0(历史每期都一样)时本期一样判常规、不一样判明显超出。本期的值(不论 basis)是历史各期里最高 / 最低的
|
||||
@@ -29,7 +31,8 @@ from app.analysis import comparable as cmp
|
||||
from app.analysis import periods as pd
|
||||
from app.analysis import phrases as ph
|
||||
from app.analysis import stats
|
||||
from app.analysis.common import Env, build, column_label, column_types, label_for, need, refuse, sign_word, today_param
|
||||
from app.analysis.common import (Env, build, column_label, column_types, label_for, need, refuse, reserved_check,
|
||||
sign_word, today_param)
|
||||
from app.analysis.contract import CallError
|
||||
from app.analysis.inputs import Snap, same_value
|
||||
from app.analysis.numbers import canon, to_fraction
|
||||
@@ -38,6 +41,19 @@ from app.analysis.numbers import canon, to_fraction
|
||||
MAD_SCALE = Fraction(14826, 10000)
|
||||
#: lag 的上限(周数据同比是 52)
|
||||
_LAG_MAX = 260
|
||||
#: 结论里记的阈值
|
||||
USED: dict[str, tuple[str, ...]] = {
|
||||
"baseline": ("k_warn", "k_alert", "min_history", "max_history"),
|
||||
"trend": ("trend_min_points", "trend_alpha", "max_history"),
|
||||
}
|
||||
#: 结果表在期次列之外固定的几列(期次列不能叫这些)
|
||||
FIXED: dict[str, tuple[str, ...]] = {"baseline": ("value", "change", "role"), "trend": ("value", "change")}
|
||||
|
||||
|
||||
def refusal_basis(op: str, settings: Mapping[str, Any]) -> tuple[list[str], str]:
|
||||
"""还没算就算不了时结论里记的阈值和判定标准(同 decompose.refusal_basis)。"""
|
||||
criterion = ph.criterion_baseline if op == "baseline" else ph.criterion_trend
|
||||
return list(USED[op]), criterion(settings, None, "期")
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -50,12 +66,27 @@ class Series:
|
||||
points: list[tuple[pd.Period, Fraction]] # 按期次排好、度量不为空的
|
||||
nulls: int
|
||||
params: dict[str, Any]
|
||||
periods: dict[int, pd.Period] # 每一期(含度量为空的),按 ordinal
|
||||
empty: set[int] # 度量为空的那几期的 ordinal
|
||||
|
||||
def whole(self, item: pd.Period) -> bool:
|
||||
return pd.short_days(item, self.grain) is None
|
||||
|
||||
def missing(self, ordinal: int) -> tuple[str, str] | None:
|
||||
"""某一期拿不到能用的值的原因:(empty 值为空 / short 不到整周, 那一期的写法);真缺了这一期为 None。"""
|
||||
item = self.periods.get(ordinal)
|
||||
if item is None:
|
||||
return None
|
||||
if ordinal in self.empty:
|
||||
return "empty", ph.key_text([item.raw])
|
||||
return ("short", ph.key_text([item.raw])) if not self.whole(item) else None
|
||||
|
||||
|
||||
def _load(raw: Mapping[str, Any], env: Env, used: list[str]) -> Series | Any:
|
||||
"""读出一条序列;算不了时直接交回 refused 的结论。"""
|
||||
snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号")
|
||||
period = snap.column(need(raw, "period", "期次列"), "期次列")
|
||||
reserved_check(period, FIXED[env.op], "期次列")
|
||||
measure = snap.column(need(raw, "measure", "度量列"), "度量列")
|
||||
grain = raw.get("grain")
|
||||
if grain is not None and grain not in (*pd.GRAINS, "seq"):
|
||||
@@ -71,29 +102,45 @@ def _load(raw: Mapping[str, Any], env: Env, used: list[str]) -> Series | Any:
|
||||
criterion=criterion)
|
||||
i_period = snap.index(period)
|
||||
values = [row[i_period] if i_period < len(row) else None for row in snap.rows]
|
||||
parsed = pd.parse(values, grain)
|
||||
weeks = pd.week_scheme(snap.sql, period, snap.dialect)
|
||||
parsed = pd.parse(values, grain, weeks=weeks)
|
||||
if parsed is None:
|
||||
sample = "、".join(dict.fromkeys(ph.key_text([v]) for v in values[:3]))
|
||||
return refuse(env, params, used, "period_unreadable",
|
||||
f"期次列「{period}」的值认不出是哪一期({sample}):要日期、YYYY-MM、YYYY-Www、YYYY-Qn 或整数序号",
|
||||
return refuse(env, params, used, "period_unreadable", unreadable_text(period, values, grain, weeks),
|
||||
criterion=criterion)
|
||||
items, grain = parsed
|
||||
params["grain"] = grain
|
||||
seen: set[int] = set()
|
||||
periods: dict[int, pd.Period] = {}
|
||||
empty: set[int] = set()
|
||||
points: list[tuple[pd.Period, Fraction]] = []
|
||||
nulls = 0
|
||||
for r, item in enumerate(items):
|
||||
if item.ordinal in seen:
|
||||
if item.ordinal in periods:
|
||||
raise CallError("duplicate_rows", ph.say(f"{snap.ref} 里同一期有不止一行:在 SQL 里按 ",
|
||||
column_label(env, snap, period), " 汇总,每期一行"))
|
||||
seen.add(item.ordinal)
|
||||
periods[item.ordinal] = item
|
||||
value = snap.number(r, measure, "度量列")
|
||||
if value is None:
|
||||
nulls += 1
|
||||
empty.add(item.ordinal)
|
||||
continue
|
||||
points.append((item, value))
|
||||
points.sort(key=lambda p: p[0].ordinal)
|
||||
return Series(snap, period, measure, grain, label, points, nulls, params)
|
||||
return Series(snap, period, measure, grain, label, points, len(empty), params, periods, empty)
|
||||
|
||||
|
||||
#: 给了粒度、期次对不上时的说法
|
||||
_GRAIN_WORD = {"day": "按天", "week": "按周", "month": "按月", "quarter": "按季度", "seq": "按序号"}
|
||||
|
||||
|
||||
def unreadable_text(period: str, values: list[Any], grain: str | None, weeks: str | None) -> str:
|
||||
"""期次认不出的原因:举出认不出的那几个值(最多三个),按为什么认不出分开说(periods.unreadable)。"""
|
||||
kind, odd = pd.unreadable(values, grain, weeks=weeks)
|
||||
shown = "、".join("空值" if v is None else ph.key_text([v]) for v in odd[:3]) + ("等" if len(odd) > 3 else "")
|
||||
if kind == "mixed":
|
||||
return ph.say(f"期次列「{period}」的写法不一:", shown, "和其余的不是同一种写法")
|
||||
if kind == "week":
|
||||
return ph.say(f"期次列「{period}」里的", shown, "按 SQL 里的周号算法没有这一周")
|
||||
if kind == "grain":
|
||||
return ph.say(f"期次列「{period}」的值(", shown, f")对不上{_GRAIN_WORD.get(str(grain), '给的')}的粒度")
|
||||
return f"期次列「{period}」的值认不出是哪一期({shown}):要日期、YYYY-MM、YYYY-Www、YYYY-Qn 或整数序号"
|
||||
|
||||
|
||||
def _criterion(env: Env, n: int | None, unit: str) -> str:
|
||||
@@ -102,13 +149,11 @@ def _criterion(env: Env, n: int | None, unit: str) -> str:
|
||||
return ph.criterion_trend(env.settings, n, unit)
|
||||
|
||||
|
||||
def _completeness(env: Env, series: Series, item: pd.Period) -> tuple[str, str | None]:
|
||||
"""本期齐不齐(同可比性检查的整期):(pass / short / unknown, 不齐的那一句)。"""
|
||||
def _completeness(env: Env, series: Series, item: pd.Period) -> tuple[str, str | None, str | None]:
|
||||
"""本期齐不齐(同可比性检查的整期,comparable.period_complete):(pass / short / unknown, 不齐的那一句, 核对不了时的附带说明)。"""
|
||||
series.params["today"] = today_param(env)
|
||||
if item.start is None:
|
||||
return "unknown", None
|
||||
span = cmp.period_span("本期", series.snap, series.period, item, series.grain, series.measure, env.facts, env.book)
|
||||
return cmp.span_coverage(span, env.today, env.book) # type: ignore[arg-type]
|
||||
return cmp.period_complete(series.snap, series.period, item, series.grain, series.measure, env.facts, env.book,
|
||||
env.today) # type: ignore[arg-type]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -117,7 +162,7 @@ def _completeness(env: Env, series: Series, item: pd.Period) -> tuple[str, str |
|
||||
|
||||
|
||||
def baseline(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["k_warn", "k_alert", "min_history", "max_history"]
|
||||
used = list(USED["baseline"])
|
||||
loaded = _load(raw, env, used)
|
||||
if not isinstance(loaded, Series):
|
||||
return loaded
|
||||
@@ -138,24 +183,26 @@ def baseline(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
found = next((p for p in series.points if same_value(raw["target"], p[0].raw)), None)
|
||||
if found is None:
|
||||
raise CallError("period_not_found", f"期次列「{series.period}」里没有「{raw['target']}」(或那一期为空):"
|
||||
"本期要写序列里有值的一期,不写就是最后一期")
|
||||
"本期要写序列里有值的一期,不写就是最后一期",
|
||||
reason=f"期次列「{series.period}」里没有本期「{ph.key_text([raw['target']])}」(或那一期为空)")
|
||||
else:
|
||||
found = series.points[-1]
|
||||
target, x_level = found
|
||||
params["target"] = target.raw
|
||||
caveats: list[dict[str, str]] = []
|
||||
status, why = _completeness(env, series, target)
|
||||
status, why, note = _completeness(env, series, target)
|
||||
if status == "short":
|
||||
return refuse(env, params, used, "partial_target", f"{why},和完整的{per}比没有意义", criterion=criterion)
|
||||
if status == "unknown" and target.start is not None:
|
||||
caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE))
|
||||
by_ordinal = {p.ordinal: v for p, v in series.points}
|
||||
before = [(p, v) for p, v in series.points if p.ordinal < target.ordinal]
|
||||
if status == "unknown" and note:
|
||||
caveats.append(ph.caveat("comparability", note))
|
||||
# 不到整周的那几期(跨年切开的)既不当比较期、也不算进历史
|
||||
by_ordinal = {p.ordinal: v for p, v in series.points if series.whole(p)}
|
||||
before = [(p, v) for p, v in series.points if p.ordinal < target.ordinal and series.whole(p)]
|
||||
cut = sum(1 for p, _ in series.points if p.ordinal < target.ordinal and not series.whole(p))
|
||||
if basis == "change":
|
||||
prev = by_ordinal.get(target.ordinal - lag)
|
||||
if prev is None:
|
||||
return refuse(env, params, used, "target_gap",
|
||||
f"本期和它的比较期(往前第 {lag} {'期' if series.grain == 'seq' else unit})之间缺期,没有比较期的值",
|
||||
return refuse(env, params, used, "target_gap", _compare_gap(series, target.ordinal - lag, lag, unit),
|
||||
criterion=criterion)
|
||||
if prev == 0:
|
||||
return refuse(env, params, used, "zero_base", ph.say("比较期的", series.label, "为 0,算不了变化率"),
|
||||
@@ -190,6 +237,8 @@ def baseline(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
caveats.append(ph.caveat("record_low", ph.record_text(False, series.label, n, unit)))
|
||||
if series.nulls:
|
||||
caveats.append(ph.caveat("null_values", ph.null_periods_text(series.nulls, series.label)))
|
||||
if cut:
|
||||
caveats.append(ph.caveat("partial_periods", ph.partial_periods_text(cut)))
|
||||
current = ph.current_text(x, basis=basis, lag=lag, grain=series.grain)
|
||||
if sigma == 0:
|
||||
shown = ph.pct(m) if basis == "change" else ph.amount(m)
|
||||
@@ -213,6 +262,17 @@ def baseline(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats)
|
||||
|
||||
|
||||
def _compare_gap(series: Series, ordinal: int, lag: int, unit: str) -> str:
|
||||
"""比较期拿不到值的原因,照实分开说:那一期的值为空、那一期不到整周、真缺了那一期。"""
|
||||
found = series.missing(ordinal)
|
||||
if found is not None and found[0] == "empty":
|
||||
return ph.say("比较期(", found[1], ")的", series.label, "为空,没有比较期的值")
|
||||
if found is not None:
|
||||
days = pd.short_days(series.periods[ordinal], series.grain)
|
||||
return ph.say("比较期(", found[1], f")只有 {days} 天,不是整周,和它比没有意义")
|
||||
return f"本期和它的比较期(往前第 {lag} {'期' if series.grain == 'seq' else unit})之间缺期,没有比较期的值"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 趋势
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -233,7 +293,7 @@ def run_length(values: list[Fraction]) -> tuple[int, str | None]:
|
||||
|
||||
|
||||
def trend(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
used = ["trend_min_points", "trend_alpha", "max_history"]
|
||||
used = list(USED["trend"])
|
||||
loaded = _load(raw, env, used)
|
||||
if not isinstance(loaded, Series):
|
||||
return loaded
|
||||
@@ -245,15 +305,15 @@ def trend(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
return refuse(env, params, used, "history_short", f"只有 0 期,至少要 {minimum} 期", criterion=criterion)
|
||||
last = series.points[-1][0]
|
||||
caveats: list[dict[str, str]] = []
|
||||
status, why = _completeness(env, series, last)
|
||||
status, why, note = _completeness(env, series, last)
|
||||
if status == "short":
|
||||
return refuse(env, params, used, "partial_target", f"{why},和完整的{per}比没有意义", criterion=criterion)
|
||||
if status == "unknown" and last.start is not None:
|
||||
caveats.append(ph.caveat("comparability", ph.UNVERIFIED_COMPLETE))
|
||||
if status == "unknown" and note:
|
||||
caveats.append(ph.caveat("comparability", note))
|
||||
cap = int(env.settings["max_history"]) + 1
|
||||
tail = [series.points[-1]]
|
||||
for point in reversed(series.points[:-1]):
|
||||
if len(tail) >= cap or point[0].ordinal != tail[-1][0].ordinal - 1:
|
||||
if len(tail) >= cap or point[0].ordinal != tail[-1][0].ordinal - 1 or not series.whole(point[0]):
|
||||
break
|
||||
tail.append(point)
|
||||
tail.reverse()
|
||||
@@ -263,7 +323,8 @@ def trend(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
gap = len(series.points) > n and len(tail) < cap
|
||||
if gap:
|
||||
return refuse(env, params, used, "series_gap",
|
||||
f"最后连着的只有 {n} 期,再往前缺期(至少要连着的 {minimum} 期)", criterion=criterion)
|
||||
f"最后连着的只有 {n} 期,{_tail_gap(series, tail[0][0].ordinal - 1)}(至少要连着的 {minimum} 期)",
|
||||
criterion=criterion)
|
||||
return refuse(env, params, used, "history_short", f"只有 {n} 期,至少要 {minimum} 期", criterion=criterion)
|
||||
ys = [v for _, v in tail]
|
||||
xs = [Fraction(i) for i in range(n)]
|
||||
@@ -302,4 +363,14 @@ def trend(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
phrase=ph.with_caveats(main, caveats, skip=("comparability",)), caveats=caveats)
|
||||
|
||||
|
||||
__all__ = ["MAD_SCALE", "baseline", "run_length", "trend"]
|
||||
def _tail_gap(series: Series, ordinal: int) -> str:
|
||||
"""最后连着的一段往前断开的原因:那一期的值为空、不到整周,或者真缺了那一期。"""
|
||||
found = series.missing(ordinal)
|
||||
if found is not None and found[0] == "empty":
|
||||
return ph.say("再往前的", found[1], "那一期的", series.label, "为空")
|
||||
if found is not None:
|
||||
return ph.say("再往前的", found[1], f"只有 {pd.short_days(series.periods[ordinal], series.grain)} 天,不是整周")
|
||||
return "再往前缺期"
|
||||
|
||||
|
||||
__all__ = ["FIXED", "MAD_SCALE", "USED", "baseline", "refusal_basis", "run_length", "trend", "unreadable_text"]
|
||||
|
||||
+105
-16
@@ -1,5 +1,8 @@
|
||||
"""测试用的分析结论(只给测试用):第 1 波各包并行开发时,证据接入、入口、说法核对的测试不等算子写完,按契约的形状
|
||||
拿一份假的结论。算子落地后,端到端的测试改用 analysis.run 真算。"""
|
||||
拿一份假的结论。算子落地后,端到端的测试改用 analysis.run 真算。
|
||||
|
||||
每个算子的假结论带它自己形状的结果表、具名标量、参数和输入(列 id、叫法、类型照各算子的实现:变化拆解按维度、波动基线和
|
||||
趋势按期次、差异检验按对、可比性检查按检查项),数是编的、彼此对得上判定语。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
@@ -18,14 +21,11 @@ _PHRASES = {
|
||||
"incomparable": "两期不可比:本期只有 3 天数据",
|
||||
}
|
||||
|
||||
_WEEKS = ["2026-07-20", "2026-07-27", "2026-08-03", "2026-08-10", "2026-08-17", "2026-08-24", "2026-08-31",
|
||||
"2026-09-07", "2026-09-14", "2026-09-21", "2026-09-28", "2026-10-05", "2026-10-12"]
|
||||
|
||||
def fake_result(op: str = "decompose", mode: str | None = "additive", verdict: str | None = "driver", *,
|
||||
subjects: list[list[str]] | None = None, direction: str | None = "down", status: str = "ok",
|
||||
reason_code: str | None = None, inputs: list[dict[str, Any]] | None = None,
|
||||
**overrides: Any) -> AnalysisResult:
|
||||
"""一份形状完整的假结论。缺省是「销售额下降主要来自华东」的可加拆解。"""
|
||||
if op not in ("decompose", "differ"):
|
||||
mode = None
|
||||
|
||||
def _decompose() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]:
|
||||
table = {
|
||||
"columns": ["region", "base", "target", "delta", "pct", "share", "flag"],
|
||||
"labels": {"region": "地区", "base": "基期", "target": "本期", "delta": "变化", "pct": "变化率",
|
||||
@@ -35,26 +35,115 @@ def fake_result(op: str = "decompose", mode: str | None = "additive", verdict: s
|
||||
["华南", "700", "666", "-34", "-0.0486", "0.1753", None],
|
||||
["西南", "100", "100", "0", "0", "0", None]],
|
||||
}
|
||||
values = {"base_total": "2220", "target_total": "2026", "delta_total": "-194", "pct_total": "-0.0874",
|
||||
"top_key": "华东", "top_share": "0.6186", "segments": "4"}
|
||||
params = {"query": "Q2", "by": ["region"], "period": "week", "base": "W41", "target": "W42", "measure": "gmv",
|
||||
"label": "销售额"}
|
||||
return table, values, params, ["region", "week", "gmv"]
|
||||
|
||||
|
||||
def _baseline() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]:
|
||||
changes = ["0.012", "-0.021", "0.034", "-0.008", "0.019", "-0.045", "0.027", "-0.015", "0.006", "0.041", "-0.032",
|
||||
"-0.004"]
|
||||
levels = ["2210", "2164", "2238", "2220", "2262", "2160", "2218", "2185", "2198", "2288", "2215", "2206"]
|
||||
columns = ["week", "value", "change", "role"]
|
||||
rows = [[w, v, c, "历史"] for w, v, c in zip(_WEEKS[:-1], levels, changes)] + [[_WEEKS[-1], "2138", "-0.031", "本期"]]
|
||||
table = {"columns": columns, "labels": {"week": "周", "value": "销售额", "change": "变化率", "role": "说明"},
|
||||
"rows": rows, "column_types": {"week": "text", "value": "number", "change": "number", "role": "text"}}
|
||||
values = {"x": "-0.031", "value": "2138", "median": "0.001", "band_lo": "-0.061", "band_hi": "0.053", "z": "-1.12",
|
||||
"rank": "11", "n_history": "12", "hist_min": "-0.045", "hist_max": "0.041"}
|
||||
params = {"query": "Q1", "period": "week", "measure": "gmv", "label": "销售额", "grain": "week", "basis": "change",
|
||||
"lag": 1, "target": _WEEKS[-1], "today": "2026-10-21"}
|
||||
return table, values, params, ["week", "gmv"]
|
||||
|
||||
|
||||
def _trend() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]:
|
||||
levels = [100, 103, 106, 109, 113, 116, 119, 123, 127, 130]
|
||||
rows = [[w, str(v), str(v - levels[i - 1]) if i else None] for i, (w, v) in enumerate(zip(_WEEKS[-10:], levels))]
|
||||
table = {"columns": ["week", "value", "change"], "labels": {"week": "周", "value": "线上销售额", "change": "比上一期"},
|
||||
"rows": rows, "column_types": {"week": "text", "value": "number", "change": "number"}}
|
||||
values = {"slope": "3.333333333333", "rel_slope": "0.03", "p": "0.000002755732", "s": "45", "n": "10",
|
||||
"run_length": "9", "run_direction": "up"}
|
||||
params = {"query": "Q1", "period": "week", "measure": "gmv", "label": "线上销售额", "grain": "week",
|
||||
"today": "2026-10-21"}
|
||||
return table, values, params, ["week", "gmv"]
|
||||
|
||||
|
||||
def _differ() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]:
|
||||
columns = ["a", "b", "a_value", "b_value", "diff", "ci_lo", "ci_hi", "a_n", "b_n", "test", "p", "p_adj", "flag"]
|
||||
table = {"columns": columns,
|
||||
"labels": {"a": "组", "b": "对照组", "a_value": "组的比率", "b_value": "对照组的比率", "diff": "差",
|
||||
"ci_lo": "差的区间下限", "ci_hi": "差的区间上限", "a_n": "组的样本量", "b_n": "对照组的样本量",
|
||||
"test": "检验", "p": "p 值", "p_adj": "校正后的 p 值", "flag": "说明"},
|
||||
"rows": [["西南", "华东", "0.05", "0.021739130435", "0.028260869565", "-0.026", "0.137", "40", "230",
|
||||
"Fisher 精确检验", "0.27", "0.27", "样本少,用了精确检验"]],
|
||||
"column_types": {"a": "text", "b": "text", **{c: "number" for c in columns[2:9]}, "test": "text",
|
||||
"p": "number", "p_adj": "number", "flag": "text"}}
|
||||
values = {"a": "0.05", "b": "0.021739130435", "diff": "0.028260869565", "ci_lo": "-0.026", "ci_hi": "0.137",
|
||||
"p": "0.27", "p_adj": "0.27", "n_a": "40", "n_b": "230", "test": "fisher", "pairs": "1", "differs": "0"}
|
||||
params = {"query": "Q3", "by": ["region"], "numerator": "refunds", "denominator": "orders", "label": "退款率",
|
||||
"pairs": [["西南", "华东"]]}
|
||||
return table, values, params, ["region", "refunds", "orders"]
|
||||
|
||||
|
||||
def _comparable() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]:
|
||||
columns = ["check", "status", "base", "target", "note"]
|
||||
table = {"columns": columns,
|
||||
"labels": {"check": "检查项", "status": "结果", "base": "基期", "target": "本期", "note": "说明"},
|
||||
"rows": [["同口径", "通过", None, None, None],
|
||||
["窗口长度", "通过", "2026-10-05 到 2026-10-11(7 天)", "2026-10-12 到 2026-10-18(7 天)", None],
|
||||
["整期", "通过", "通过", "通过", None], ["星期构成", "通过", None, None, None],
|
||||
["数据问题", "通过", None, None, None]],
|
||||
"column_types": dict.fromkeys(columns, "text")}
|
||||
values = {"same_query": "pass", "window": "pass", "complete": "pass", "weekdays": "pass", "data": "pass",
|
||||
"base_start": "2026-10-05", "base_end": "2026-10-11", "base_days": "7", "base_weekend": "2",
|
||||
"target_start": "2026-10-12", "target_end": "2026-10-18", "target_days": "7", "target_weekend": "2"}
|
||||
params = {"base_query": "Q4", "target_query": "Q5", "measure": "gmv", "today": "2026-10-21"}
|
||||
return table, values, params, ["gmv"]
|
||||
|
||||
|
||||
_SHAPES = {"decompose": _decompose, "baseline": _baseline, "trend": _trend, "differ": _differ,
|
||||
"comparable": _comparable}
|
||||
_CRITERIA = {"decompose": "一段占总变化一半以上算主要来源",
|
||||
"baseline": "本期和过去 12 周的中位数比,差不到 2 倍稳健标准差算常规波动",
|
||||
"trend": "Mann–Kendall 检验的双侧 p 值小于 0.05 时按 Theil–Sen 斜率的正负判上升或下降",
|
||||
"differ": "双侧 p 值小于 0.05 算差异站得住",
|
||||
"comparable": "只差日期、窗口一样长、是整期、星期构成相同才算可比"}
|
||||
|
||||
|
||||
def fake_result(op: str = "decompose", mode: str | None = "additive", verdict: str | None = "driver", *,
|
||||
subjects: list[list[str]] | None = None, direction: str | None = "down", status: str = "ok",
|
||||
reason_code: str | None = None, inputs: list[dict[str, Any]] | None = None,
|
||||
**overrides: Any) -> AnalysisResult:
|
||||
"""一份形状完整的假结论。缺省是「销售额下降主要来自华东」的可加拆解;别的算子给各自形状的结果表、具名标量和参数。"""
|
||||
if op not in ("decompose", "differ"):
|
||||
mode = None
|
||||
elif op == "differ" and mode not in ("rate", "mean", "rows"):
|
||||
mode = "rate"
|
||||
table, values, params, columns = _SHAPES.get(op, _decompose)()
|
||||
if op == "comparable":
|
||||
default_inputs = [{"ref": ref, "artifact": "0" * 64, "role": role, "columns": columns, "rows": "all"}
|
||||
for ref, role in (("Q4", "base"), ("Q5", "target"))]
|
||||
else:
|
||||
default_inputs = [{"ref": str(params.get("query") or "Q2"), "artifact": "0" * 64, "role": "data",
|
||||
"columns": columns, "rows": "all"}]
|
||||
result: AnalysisResult = {
|
||||
"op": op,
|
||||
"mode": mode,
|
||||
"method": METHODS.get((op, mode), f"{op}@1"),
|
||||
"params": {"query": "Q2", "by": ["region"], "period": "week", "base": "W41", "target": "W42",
|
||||
"measure": "gmv", "label": "销售额"},
|
||||
"params": params,
|
||||
"settings": effective_settings(None),
|
||||
"inputs": inputs if inputs is not None else [
|
||||
{"ref": "Q2", "artifact": "0" * 64, "role": "data", "columns": ["region", "week", "gmv"], "rows": "all"}],
|
||||
"inputs": inputs if inputs is not None else default_inputs,
|
||||
"probes": [],
|
||||
"status": status, # type: ignore[typeddict-item]
|
||||
"reason_code": reason_code,
|
||||
"reason": None if status == "ok" else "历史只有 5 期,至少要 8 期",
|
||||
"table": table if status == "ok" else None,
|
||||
"values": {"base_total": "2220", "target_total": "2026", "delta_total": "-194", "pct_total": "-0.0874",
|
||||
"top_key": "华东", "top_share": "0.6186", "segments": "4"} if status == "ok" else {},
|
||||
"table": table if status == "ok" else None, # type: ignore[typeddict-item]
|
||||
"values": values if status == "ok" else {},
|
||||
"verdict": ({"code": verdict, "subjects": subjects if subjects is not None else
|
||||
([["华东"]] if verdict == "driver" else []), "direction": direction} # type: ignore[typeddict-item]
|
||||
if status == "ok" and verdict else None),
|
||||
"criterion": "一段占总变化一半以上算主要来源",
|
||||
"criterion": _CRITERIA.get(op, _CRITERIA["decompose"]),
|
||||
"phrase": _PHRASES.get(verdict or "", "") if status == "ok" else "变化拆解没有做成:历史只有 5 期,至少要 8 期",
|
||||
"caveats": [],
|
||||
"hypothesis": "地区拖累",
|
||||
|
||||
@@ -907,13 +907,14 @@ def _analysis_item_ok(key: str, value: Any) -> bool:
|
||||
|
||||
def analysis_values(stored: Any) -> dict[str, Any]:
|
||||
"""库里存的(或刚写进来的)analysis 一组 → 补齐缺省;存坏了的项按缺省。两两之间的大小对不上的(「明显超出」不大于
|
||||
「超出常规波动」、最多用几期历史小于至少几期)那一对都按缺省:只换掉一项,可能换出另一种对不上。"""
|
||||
「超出常规波动」、最多用几期历史小于至少几期、「集中」的合计占比小于「主要来源」的占比)那一对都按缺省:只换掉一项,可能
|
||||
换出另一种对不上。"""
|
||||
stored = stored if isinstance(stored, dict) else {}
|
||||
out = dict(analysis_contract.DEFAULTS)
|
||||
for key in analysis_contract.DEFAULTS:
|
||||
if key in stored and _analysis_item_ok(key, stored[key]):
|
||||
out[key] = float(stored[key]) if analysis_contract.RANGES[key][0] is float else stored[key]
|
||||
for pair in (("k_warn", "k_alert"), ("min_history", "max_history")):
|
||||
for pair in (("k_warn", "k_alert"), ("min_history", "max_history"), ("main_share", "concentration")):
|
||||
if analysis_contract.settings_problem({k: out[k] for k in pair}) is not None:
|
||||
out.update({k: analysis_contract.DEFAULTS[k] for k in pair})
|
||||
return out
|
||||
|
||||
@@ -13,7 +13,10 @@ driver……),下游条件分支能按它走(周报「超出常规波动
|
||||
(contract.ledger_entry,带 item),报告的写作目录编成 A{n};每项另发一条 analysis.end(运行详情、运行对比按项对齐)。
|
||||
|
||||
**一项算不了不让卡失败。** 算不了(历史不够、本期不完整、截断……)就是这一项的结论(status refused),照实记、发一条
|
||||
analysis_refused 提醒,接着做下一项。调用写错(列不存在、比率当可加……)是模板的毛病:整张卡失败,说清是哪一项、怎么改。
|
||||
analysis_refused 提醒,接着做下一项。模板定死了调用,这期的数据里没有要用的那一项(上一层没做成、下钻的段这期没有、期次
|
||||
这期没有、总数那一格越界或为空、要比的组这期没有,analysis.DATA_ERRORS)不是写错了,同样记成算不了的一项
|
||||
(run 的 refuse_data_errors)。结构性的调用错误(列不存在、遮罩列、比率当可加、重复行、参数写错……)是模板的毛病:整张卡
|
||||
失败,说清是哪一项、怎么改。
|
||||
|
||||
分析的结果不作口径卡的输入(D16):口径卡读 nodes.<卡>.results 的,按追不到原始数据处理(受管级发布时另有提示)。
|
||||
"""
|
||||
@@ -53,7 +56,12 @@ async def run_analysis(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
if errors := [i for i in analysis_issues(ctx.run.spec, ctx.node) if i.level == "error"]:
|
||||
raise NodeError(ctx.node.id, errors[0].message)
|
||||
items: list[dict[str, Any]] = ctx.config["items"]
|
||||
settings = await settings_of(run_snapshot(ctx.run), ctx.config.get("overrides") or None)
|
||||
overrides = ctx.config.get("overrides") or None
|
||||
settings = await settings_of(run_snapshot(ctx.run), overrides)
|
||||
# 画图时阈值覆盖只能对着缺省值校验;合并了这次运行的设置快照以后再核一遍(管理员把「超出常规波动」调到 3、卡上把「明显
|
||||
# 超出」覆盖成 2.5,就永远判不出超出)。这是配置错误,不是数据问题:让卡失败、说清冲突在哪
|
||||
if overrides and (problem := contract.settings_problem(settings)):
|
||||
raise NodeError(ctx.node.id, f"阈值覆盖和这次运行的分析设置冲突:{problem}。请改分析卡上的阈值覆盖")
|
||||
|
||||
queries = [e for e in state.get("evidence") or [] if isinstance(e, dict) and e.get("kind") == "query"]
|
||||
inputs: dict[str, InputSnapshot] = {}
|
||||
@@ -74,8 +82,9 @@ async def run_analysis(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
who = f"分析项「{item_id}」({ANALYSIS_OP_LABEL[op]})"
|
||||
request = _request(item, state, ctx, who)
|
||||
try:
|
||||
# 下钻(parent)只认这张卡里排在前面、已经做完的项:done 里此刻只有它们
|
||||
result = await analysis_ops.run(request, inputs.get, settings=settings, context=context, parents=done.get)
|
||||
# 下钻(parent)只认这张卡里排在前面、已经做完的项:done 里此刻只有它们(算不了的也在,下钻那一项照实记成算不了)
|
||||
result = await analysis_ops.run(request, inputs.get, settings=settings, context=context, parents=done.get,
|
||||
refuse_data_errors=True)
|
||||
except CallError as e:
|
||||
raise NodeError(ctx.node.id, f"{who}:{e.message}。请在分析卡中改正这一项") from None
|
||||
except Exception as e: # noqa: BLE001 - 算子本身出错:说清是哪一项,原因照实
|
||||
|
||||
@@ -52,8 +52,10 @@ CATALOGS = {"shop": {"orders": ORDERS}}
|
||||
async def analyze(op: str, mode: str | None = None, *, snaps: Sequence[Mapping[str, Any]], today: date = TODAY,
|
||||
catalogs: Mapping[str, Any] | None = None, probe: Any = None, facts: Any = None,
|
||||
parents: Mapping[str, Mapping[str, Any]] | None = None, settings: Mapping[str, Any] | None = None,
|
||||
hypothesis: str | None = None, check: bool = True, **params: Any) -> dict[str, Any]:
|
||||
"""跑一次 analysis.run;check 时再复算一遍,核对和封存的逐字段相同(hypothesis 除外)。"""
|
||||
hypothesis: str | None = None, check: bool = True, refuse_data_errors: bool = False,
|
||||
**params: Any) -> dict[str, Any]:
|
||||
"""跑一次 analysis.run;check 时再复算一遍,核对和封存的逐字段相同(hypothesis 除外)。refuse_data_errors 同 run(分析卡
|
||||
传真);复算一律按真。"""
|
||||
by_ref = {s["ref"]: s for s in snaps}
|
||||
request: dict[str, Any] = {"op": op, "mode": mode, "params": params}
|
||||
if hypothesis is not None:
|
||||
@@ -64,7 +66,7 @@ async def analyze(op: str, mode: str | None = None, *, snaps: Sequence[Mapping[s
|
||||
if facts is not None:
|
||||
context["facts"] = facts
|
||||
result = await analysis.run(request, by_ref.get, settings=effective_settings(settings), context=context,
|
||||
parents=(parents or {}).get)
|
||||
parents=(parents or {}).get, refuse_data_errors=refuse_data_errors)
|
||||
if check:
|
||||
again = recomputed(result, snaps, parents)
|
||||
assert same(result, again), _diff(result, again)
|
||||
|
||||
+2
-2
@@ -30,7 +30,7 @@ class FakeOps:
|
||||
monkeypatch.setattr(analysis_ops, "run", self.run)
|
||||
|
||||
async def run(self, request: dict[str, Any], resolve: Any, *, settings: Any, context: Any,
|
||||
parents: Any = None) -> dict[str, Any]:
|
||||
parents: Any = None, refuse_data_errors: bool = False) -> dict[str, Any]:
|
||||
params = request.get("params") or {}
|
||||
names = [params[k] for k in ("query", "base_query", "target_query") if params.get(k)]
|
||||
snapshots = {name: resolve(name) for name in names}
|
||||
@@ -38,7 +38,7 @@ class FakeOps:
|
||||
if isinstance(params.get("parent"), dict) and parents is not None:
|
||||
parent = parents(params["parent"]["of"])
|
||||
self.calls.append({"request": request, "snapshots": snapshots, "settings": dict(settings), "context": context,
|
||||
"parent": parent, "parents": parents,
|
||||
"parent": parent, "parents": parents, "refuse_data_errors": refuse_data_errors,
|
||||
"visible": {n: parents is not None and parents(n) is not None for n in self.peek}})
|
||||
error = self.errors.pop(0) if self.errors else None
|
||||
if error is not None:
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""分析卡里随数据变化的调用错误(复核 R6):模板定死了调用,这期的数据里没有要用的那一项(上一层没做成、下钻的段这期没有……)
|
||||
记成算不了的一项,不让卡失败;结构性的调用错误照旧让卡失败并说清哪一项。接真算子跑(tests/test_analysis_e2e 的虚构零售库
|
||||
shop),每份结论按封存的工件复算、逐字段相同。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.engine.runner import run_manager
|
||||
from tests.test_analysis_e2e import ( # noqa: F401 - engine_up、shop 是自动用的夹具
|
||||
BY_REGION, EAST, TOOL, TWO_WEEKS, WEEK, chain, engine_up, events, node, recomputes, shop, wait,
|
||||
)
|
||||
|
||||
ONLY_EAST = (f"SELECT region, {WEEK} AS week, SUM(amount) AS gmv FROM orders "
|
||||
"WHERE region = '华东' AND day >= '2026-05-11' GROUP BY region, week ORDER BY region, week")
|
||||
WHERE = {"id": "where", "op": "decompose", "mode": "additive", "from": "by_region",
|
||||
"params": {"by": ["region"], **TWO_WEEKS}}
|
||||
DRILL = {"id": "east", "op": "decompose", "mode": "additive", "from": "east_q",
|
||||
"params": {"by": ["channel"], **TWO_WEEKS, "parent": {"of": "where", "key": ["华东"]}}}
|
||||
|
||||
|
||||
def card_graph(region_sql, items):
|
||||
return chain(node("start", "input"), node("by_region", "tool", tool=TOOL, args={"sql": region_sql}),
|
||||
node("east_q", "tool", tool=TOOL, args={"sql": EAST}),
|
||||
node("card", "analysis", items=items),
|
||||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.card.results.east.phrase }}"}]))
|
||||
|
||||
|
||||
async def _card(region_sql, items):
|
||||
run = await run_manager.start(graph=card_graph(region_sql, items), input_payload={})
|
||||
row = await wait(run.id)
|
||||
return row
|
||||
|
||||
|
||||
async def test_a_drill_under_a_refused_parent_is_refused_and_the_card_goes_on():
|
||||
row = await _card(ONLY_EAST, [WHERE, DRILL])
|
||||
assert row.status == "succeeded", row.error
|
||||
[finished] = await events(row.id, "node.finished", "card")
|
||||
results = artifact_store.load(finished["artifact"])["results"]
|
||||
assert results["where"]["status"] == "refused" and results["east"]["status"] == "refused"
|
||||
east = artifact_store.load(results["east"]["artifact"])
|
||||
assert east["reason_code"] == "parent_refused"
|
||||
assert east["reason"] == "上一层「where」没有做成(按 region 只有 1 段,至少要 2 段才拆得开),不能在它下面下钻"
|
||||
assert row.output["r"] == "变化拆解没有做成:" + east["reason"]
|
||||
ends = {e["item"]: e for e in await events(row.id, "analysis.end", "card")}
|
||||
assert ends["east"]["reason_code"] == "parent_refused" and ends["east"]["status"] == "refused"
|
||||
warned = [e for e in await events(row.id, "log", "card") if e.get("code") == "analysis_refused"]
|
||||
assert [e["item"] for e in warned] == ["where", "east"]
|
||||
assert [(e["item"], e["status"]) for e in finished["evidence"]] == [("where", "refused"), ("east", "refused")]
|
||||
recomputes(artifact_store.load(results["where"]["artifact"]))
|
||||
recomputes(east)
|
||||
|
||||
|
||||
async def test_a_segment_missing_this_time_is_refused_and_the_card_goes_on():
|
||||
drill = {**DRILL, "params": {**DRILL["params"], "parent": {"of": "where", "key": ["东北"]}}}
|
||||
row = await _card(BY_REGION, [WHERE, drill])
|
||||
assert row.status == "succeeded", row.error
|
||||
[finished] = await events(row.id, "node.finished", "card")
|
||||
results = artifact_store.load(finished["artifact"])["results"]
|
||||
assert results["where"]["status"] == "ok" and results["east"]["status"] == "refused"
|
||||
east = artifact_store.load(results["east"]["artifact"])
|
||||
assert east["reason_code"] == "input_unavailable" and east["reason"] == "上一层「where」里没有「东北」这一段"
|
||||
assert [i["role"] for i in east["inputs"]] == ["parent", "data"]
|
||||
recomputes(east)
|
||||
|
||||
|
||||
async def test_a_structural_call_error_still_fails_the_card_and_names_the_item():
|
||||
wrong = {**WHERE, "params": {**WHERE["params"], "by": ["channel"]}}
|
||||
row = await _card(BY_REGION, [wrong])
|
||||
assert row.status == "failed"
|
||||
assert "分析项「where」(变化拆解)" in row.error and "没有「channel」这一列" in row.error
|
||||
assert "请在分析卡中改正这一项" in row.error
|
||||
|
||||
|
||||
async def test_overrides_that_conflict_with_the_run_settings_fail_the_card():
|
||||
# 画图时对着缺省值校验能过(缺省「超出常规波动」是 2);这次运行的设置把它调到 3,卡上覆盖的「明显超出」2.5 就永远判不出
|
||||
card = node("card", "analysis", items=[WHERE], overrides={"k_alert": 2.5})
|
||||
graph = chain(node("start", "input"), node("by_region", "tool", tool=TOOL, args={"sql": BY_REGION}), card)
|
||||
run = await run_manager.start(graph=graph, input_payload={}, options={"analysis": {"k_warn": 3.0}})
|
||||
row = await wait(run.id)
|
||||
assert row.status == "failed"
|
||||
assert "阈值覆盖和这次运行的分析设置冲突:「明显超出」的倍数要大于「超出常规波动」的倍数" in row.error
|
||||
assert await events(row.id, "analysis.end", "card") == []
|
||||
ok = await run_manager.start(graph=graph, input_payload={}, options={"analysis": {"k_warn": 2.0}})
|
||||
assert (await wait(ok.id)).status == "succeeded"
|
||||
@@ -181,3 +181,78 @@ async def test_a_masked_date_column_is_neither_used_nor_probed():
|
||||
period("Q2", "2026-10-12", "2026-10-19", masked=["order_date"]), probe=probe, measure="gmv")
|
||||
assert probe.calls == [] and result["probes"] == []
|
||||
assert result["verdict"]["code"] == "comparable" and "窗口长度" in result["phrase"]
|
||||
|
||||
|
||||
async def test_one_snapshot_of_sqlite_weeks_uses_the_real_days_of_each_week():
|
||||
# SQLite 的 strftime('%Y-W%W'):2026-W42 是 10-19 到 10-25(运行当天 10-21 还没过完),跨年的 2025-W52、2026-W00 只有 3 天、4 天
|
||||
sql = "SELECT strftime('%Y-W%W', order_date) AS week, COUNT(*) AS orders FROM orders GROUP BY week"
|
||||
rows = [["2025-W52", 300], ["2026-W00", 400], ["2026-W01", 700], ["2026-W41", 700], ["2026-W42", 200]]
|
||||
s = snap("Q1", sql, ["week", "orders"], rows, types={"week": "text", "orders": "number"})
|
||||
probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")})
|
||||
now = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="week", base="2026-W41",
|
||||
target="2026-W42")
|
||||
assert now["verdict"]["code"] == "incomparable"
|
||||
assert now["phrase"] == "两期不可比:本期数据只到 2026-10-20,还差 5 天"
|
||||
assert now["values"]["target_start"] == "2026-10-19" and now["values"]["target_end"] == "2026-10-25"
|
||||
cut = await analyze("comparable", snaps=[s], probe=probe, query="Q1", period="week", base="2025-W52",
|
||||
target="2026-W00")
|
||||
assert cut["verdict"]["code"] == "incomparable"
|
||||
assert "两段的天数不同:3 天对 4 天" in cut["phrase"]
|
||||
assert "本期只有 2026-01-01 到 2026-01-04 这 4 天,不是整周" in comparable_reasons(cut)
|
||||
assert checks(cut)["星期构成"] == "提醒"
|
||||
# 不知道方言(认不出周号算法):窗口、整期、星期构成都核对不了,不判可比以外的结论
|
||||
plain = snap("Q1", sql, ["week", "orders"], rows, types={"week": "text", "orders": "number"})
|
||||
unknown = await analyze("comparable", snaps=[plain], query="Q1", period="week", base="2026-W41", target="2026-W42")
|
||||
assert unknown["phrase"] == "两期可比,窗口长度、整期、星期构成核对不了"
|
||||
|
||||
|
||||
def _same(a, b):
|
||||
from app.analysis.comparable import same_query
|
||||
from app.analysis.inputs import Snap
|
||||
|
||||
return same_query(Snap("Q1", {"sql": a, "source": "shop"}, "sqlite"),
|
||||
Snap("Q2", {"sql": b, "source": "shop"}, "sqlite"))
|
||||
|
||||
|
||||
def test_the_same_query_ignores_the_outer_order_and_how_run_day_bounds_are_written():
|
||||
by_region = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '{a}' AND '{b}' "
|
||||
"GROUP BY region{tail}")
|
||||
# 只有一边带 ORDER BY:排序不改结果
|
||||
check = _same(by_region.format(a="2026-10-05", b="2026-10-11", tail=""),
|
||||
by_region.format(a="2026-10-12", b="2026-10-18", tail=" ORDER BY gmv DESC"))
|
||||
assert check.status == "pass" and check.reasons == []
|
||||
# 按运行当天现算的界写法不一样(上周对本周到今天)
|
||||
window = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= {a} AND order_date < {b}"
|
||||
assert _same(window.format(a="date('now', 'weekday 1', '-14 days')", b="date('now', 'weekday 1', '-7 days')"),
|
||||
window.format(a="date('now', 'weekday 1', '-7 days')", b="date('now')")).status == "pass"
|
||||
assert _same(window.format(a="date('now', 'weekday 1', '-14 days')", b="date('now', 'weekday 1', '-7 days')"),
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', 'weekday 1', '-7 days')"
|
||||
).status == "pass"
|
||||
|
||||
|
||||
def test_what_still_makes_two_queries_different():
|
||||
base = "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' GROUP BY region"
|
||||
# 带 LIMIT 时排序决定取哪几段
|
||||
top = _same(base + " ORDER BY gmv DESC LIMIT 2", base.replace("2026-10-05", "2026-10-12") + " LIMIT 2")
|
||||
assert top.status == "fail"
|
||||
# 除了日期还差别的筛选、按别的时间列、多一个条件
|
||||
other = _same(base.replace("WHERE", "WHERE region = '华东' AND"),
|
||||
base.replace("WHERE", "WHERE region = '华南' AND").replace("2026-10-05", "2026-10-12"))
|
||||
assert (other.status, other.reasons) == ("fail", ["两边除了日期,筛选的还不一样"])
|
||||
paid = _same(base, base.replace("order_date", "paid_at").replace("2026-10-05", "2026-10-12"))
|
||||
assert paid.status == "fail" and paid.reasons == ["两份查询的口径结构不同(差在时间列)"]
|
||||
extra = _same(base, base.replace("GROUP BY", "AND status = 'paid' GROUP BY").replace("2026-10-05", "2026-10-12"))
|
||||
assert extra.status == "fail" and "筛选条件" in extra.reasons[0]
|
||||
|
||||
|
||||
async def test_decomposing_two_snapshots_that_differ_only_in_order_is_not_refused():
|
||||
a = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '2026-10-05' AND '2026-10-11' "
|
||||
"GROUP BY region")
|
||||
b = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '2026-10-12' AND '2026-10-18' "
|
||||
"GROUP BY region ORDER BY gmv DESC")
|
||||
base = snap("Q1", a, ["region", "gmv"], [["华东", 820], ["华南", 700], ["华北", 600]], types={"region": "text"})
|
||||
target = snap("Q2", b, ["region", "gmv"], [["华东", 700], ["华南", 666], ["华北", 560]], types={"region": "text"})
|
||||
result = await analyze("decompose", "additive", snaps=[base, target], probe=fixed_prober(RANGE), base_query="Q1",
|
||||
target_query="Q2", by=["region"], measure="gmv", label="销售额")
|
||||
assert result["status"] == "ok" and result["verdict"]["code"] == "driver"
|
||||
assert not [c for c in result["caveats"] if c["code"] == "comparability"]
|
||||
|
||||
@@ -19,6 +19,10 @@ def test_settings_problems_are_spoken_with_labels_not_keys():
|
||||
assert c.settings_problem({"main_share": True}) == "「主要来源」的占比要填数字"
|
||||
assert c.settings_problem({"main_share": 0.95}) == "「主要来源」的占比要在 0.3 到 0.9 之间"
|
||||
assert c.settings_problem({"max_history": 8, "min_history": 9}) == "最多用几期历史不能小于至少几期历史"
|
||||
# 「主要」要的占比比「集中」的合计还高:一段占 60% 会判成分散,判定语却说「没有哪一段占到 90%」
|
||||
assert c.settings_problem({"main_share": 0.9, "concentration": 0.5}) == "「集中」的合计占比不能小于「主要来源」的占比"
|
||||
assert c.settings_problem({"main_share": 0.85}) == "「集中」的合计占比不能小于「主要来源」的占比" # 缺省的 0.8 补齐了比
|
||||
assert c.settings_problem({"main_share": 0.8}) is None and c.settings_problem({"concentration": 0.5}) is None
|
||||
assert c.settings_problem({"nope": 1}) == "不认识的分析设置:nope"
|
||||
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
|
||||
from app.analysis import DATA_ERRORS
|
||||
from app.analysis.decompose import lmdi, ratio_effects
|
||||
from tests.analysis_samples import analyze, call_error, fixed_prober, notes, snap
|
||||
|
||||
@@ -56,8 +57,10 @@ async def test_a_driver_is_named_with_its_share_and_the_whole_shape_is_filled():
|
||||
"columns": ["region", "week", "gmv"], "rows": "all"}]
|
||||
assert result["hypothesis"] == "华东拖累" and result["probes"] == []
|
||||
assert result["criterion"].endswith("以各段之和为总数")
|
||||
assert {c["code"] for c in result["caveats"]} == {"headline_basis"}
|
||||
# 期次是现成的一列「2026-W41」,认不出是按哪种周号算的:本期齐不齐核对不了,只附在证据面板(不进判定语)
|
||||
assert {c["code"] for c in result["caveats"]} == {"headline_basis", "comparability"}
|
||||
assert result["params"]["catalog"] == {"labels": {"region": "地区"}, "measures": {"gmv": "flow"}}
|
||||
assert result["params"]["today"] == "2026-10-21"
|
||||
|
||||
|
||||
async def test_the_segments_add_up_to_the_total_digit_for_digit():
|
||||
@@ -127,6 +130,29 @@ async def test_a_rounded_headline_closes_within_its_written_precision():
|
||||
assert (result["verdict"]["code"] != "unclosed") is closes, written
|
||||
|
||||
|
||||
async def test_a_headline_rounded_by_the_database_to_a_whole_number_closes_within_half_a_unit():
|
||||
# SQLite 的 ROUND() 交回浮点 2120.0:取到了个位,最短写法里的「.0」不是一位小数的书写精度
|
||||
rows = [["华东", "2026-W41", "1120.2"], ["华东", "2026-W42", "700"], ["华北", "2026-W41", "1000.2"],
|
||||
["华北", "2026-W42", "600"]]
|
||||
for written, closes in ((2120.0, True), (2121.0, False), (2120.5, False)):
|
||||
head = snap("Q1", "SELECT ROUND(SUM(amount)) AS gmv FROM orders", ["gmv_base", "gmv"], [[written, 1300.0]])
|
||||
result = await additive(rows, snaps=[weekly(rows), head], headline=["Q1.r0.gmv_base", "Q1.r0.gmv"])
|
||||
assert (result["verdict"]["code"] != "unclosed") is closes, written
|
||||
|
||||
|
||||
async def test_a_drill_closes_against_the_exact_value_of_the_parent_segment():
|
||||
top = await additive() # 华东 820 → 700
|
||||
near = snap("Q4", "SELECT province, week, SUM(amount) AS gmv FROM orders WHERE region = '华东' GROUP BY 1, 2",
|
||||
["province", "week", "gmv"],
|
||||
[["沪", "2026-W41", "500.4"], ["沪", "2026-W42", "410"], ["苏", "2026-W41", 320],
|
||||
["苏", "2026-W42", "290.3"]])
|
||||
params = {"query": "Q4", "by": ["province"], "period": "week", "base": "2026-W41", "target": "2026-W42",
|
||||
"measure": "gmv", "parent": {"of": "A3", "key": ["华东"]}}
|
||||
off = await analyze("decompose", "additive", snaps=[near], parents={"A3": top}, **params)
|
||||
# 上一层那一段是算出来的精确值,不是取整过的总数:820.4 对 820、700.3 对 700 都不闭合
|
||||
assert off["verdict"]["code"] == "unclosed" and off["values"]["gap"] == "-0.4"
|
||||
|
||||
|
||||
async def test_drilling_down_checks_closure_against_the_parent_segment():
|
||||
top = await additive()
|
||||
province = snap("Q4", "SELECT province, week, SUM(amount) AS gmv FROM orders WHERE region = '华东' GROUP BY 1, 2",
|
||||
@@ -139,7 +165,7 @@ async def test_drilling_down_checks_closure_against_the_parent_segment():
|
||||
assert child["params"]["depth"] == 1 and child["params"]["parent"] == {"of": "A3", "key": ["华东"]}
|
||||
assert child["inputs"][0]["role"] == "parent" and child["inputs"][0]["ref"] == "A3"
|
||||
assert "max_drill_depth" in child["settings"]
|
||||
assert {c["code"] for c in child["caveats"]} == set() # 有 parent 就不是「以各段之和为总数」
|
||||
assert {c["code"] for c in child["caveats"]} == {"comparability"} # 有 parent 就不是「以各段之和为总数」
|
||||
short = snap("Q4", province["sql"], ["province", "week", "gmv"],
|
||||
[["沪", "2026-W41", 500], ["沪", "2026-W42", 410], ["苏", "2026-W41", 300], ["苏", "2026-W42", 290]])
|
||||
unclosed = await analyze("decompose", "additive", snaps=[short], parents={"A3": top}, **params)
|
||||
@@ -301,9 +327,12 @@ async def test_simpson_each_channel_improves_but_the_mix_makes_the_whole_worse()
|
||||
rows = [["线上", "W41", 50, 500], ["线上", "W42", 90, 1000], ["线下", "W41", 10, 1000], ["线下", "W42", 4, 500]]
|
||||
result = await ratio(rows, label="退款率")
|
||||
assert result["verdict"] == {"code": "mix_driven", "subjects": [["线上"]], "direction": "up"}
|
||||
assert result["phrase"].startswith("退款率上升 2.3 个百分点(4.0% 到 6.3%),主要来自结构变化:线上占比从 33.3% 到 66.7%")
|
||||
# 结构变化只说一遍:正文说了结构变化、点了线上,辛普森那一句只说各段和整体反着变
|
||||
assert result["phrase"] == ("退款率上升 2.3 个百分点(4.0% 到 6.3%),主要来自结构变化:线上占比从 33.3% 到 66.7%;"
|
||||
"各段的退款率大多在下降,整体却上升")
|
||||
simpson = next(c for c in result["caveats"] if c["code"] == "simpson")
|
||||
assert simpson["text"] == "各段的退款率大多在下降,整体却上升:变化来自结构,线上占比从 33.3% 升到 66.7%"
|
||||
assert simpson["text"] == "各段的退款率大多在下降,整体却上升"
|
||||
assert result["phrase"].count("结构") == 1
|
||||
values = result["values"]
|
||||
assert Fraction(values["mix_total"]) + Fraction(values["rate_total"]) == pytest.approx(Fraction(values["delta"]))
|
||||
assert result["table"]["columns"][:5] == ["channel", "n_base", "n_target", "d_base", "d_target"]
|
||||
@@ -448,3 +477,84 @@ async def test_dimension_columns_keep_the_type_recorded_at_query_time():
|
||||
factor_result = await factor([["W41", "1000", 100, "10"], ["W42", "1320", 120, "11"]])
|
||||
assert factor_result["table"]["column_types"] == {"factor": "text", "base": "number", "target": "number",
|
||||
"pct": "number", "contrib": "number", "share": "number"}
|
||||
|
||||
|
||||
|
||||
async def test_a_label_cannot_carry_a_judgment_into_the_phrase():
|
||||
cut = await additive(label="销售额,主要来自华南")
|
||||
assert cut["params"]["label"] == "销售额" and cut["phrase"].startswith("销售额下降 194")
|
||||
assert "华南" not in cut["phrase"]
|
||||
judged = await additive(label="主要来自华南的销售额")
|
||||
assert "label" not in judged["params"] # 消成空了,退回目录里的叫法
|
||||
assert judged["phrase"].startswith("销售额下降 194") and "华南" not in judged["phrase"]
|
||||
# 消过一遍会变的叫法:复算拿记下的叫法再消,判定语逐字相同(analyze 核对复算逐字段相同)
|
||||
marked = await additive(label="销售额[ [v:Q1.r0.gmv] ]")
|
||||
assert marked["params"]["label"] == "销售额" and marked["phrase"].startswith("销售额下降 194")
|
||||
|
||||
|
||||
async def test_a_dimension_named_like_a_result_column_is_a_call_error():
|
||||
rows = [["线上", "2026-W41", 820], ["线上", "2026-W42", 700], ["线下", "2026-W41", 600], ["线下", "2026-W42", 560]]
|
||||
flagged = snap("Q2", "SELECT channel AS flag, week, SUM(amount) AS gmv FROM orders GROUP BY 1, 2",
|
||||
["flag", "week", "gmv"], rows)
|
||||
error = await call_error("decompose", "additive", snaps=[flagged], query="Q2", by=["flag"], period="week",
|
||||
base="2026-W41", target="2026-W42", measure="gmv")
|
||||
assert error.code == "reserved_column" and "在 SQL 里给它换个别名" in error.message and "AS flag_name" in error.message
|
||||
assert error.code not in DATA_ERRORS # 结构性的:分析卡里照旧让卡失败
|
||||
ratio_rows = [["线上", "W41", 50, 1000], ["线上", "W42", 80, 1000], ["线下", "W41", 20, 1000], ["线下", "W42", 26, 1000]]
|
||||
mixed = snap("Q1", "SELECT channel AS Mix, week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders "
|
||||
"GROUP BY 1, 2", ["Mix", "week", "refunds", "orders"], ratio_rows)
|
||||
error = await call_error("decompose", "ratio", snaps=[mixed], query="Q1", by="Mix", period="week", base="W41",
|
||||
target="W42", numerator="refunds", denominator="orders")
|
||||
assert error.code == "reserved_column" and "「Mix」" in error.message
|
||||
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 一份快照按期次分组:本期齐不齐(同波动基线那一套)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
WEEK_START = ("SELECT region, date(order_date, '-6 days', 'weekday 1') AS week, SUM(amount) AS gmv FROM orders "
|
||||
"GROUP BY 1, 2")
|
||||
DAY_SQL = "SELECT region, order_date AS day, SUM(amount) AS gmv FROM orders GROUP BY 1, 2"
|
||||
|
||||
|
||||
def _two(base, target, *, sql=WEEK_START, col="week"):
|
||||
rows = [["华东", base, 820], ["华东", target, 700], ["华北", base, 600], ["华北", target, 560]]
|
||||
return snap("Q2", sql, ["region", col, "gmv"], rows, types={"region": "text", col: "text", "gmv": "number"})
|
||||
|
||||
|
||||
async def test_one_snapshot_by_period_checks_the_target_is_whole():
|
||||
probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")})
|
||||
ask = {"query": "Q2", "by": ["region"], "period": "week", "measure": "gmv", "label": "销售额"}
|
||||
# 运行当天 10-21:10-19 那一周没过完
|
||||
partial = await analyze("decompose", "additive", snaps=[_two("2026-10-12", "2026-10-19")], probe=probe,
|
||||
base="2026-10-12", target="2026-10-19", **ask)
|
||||
assert partial["reason_code"] == "partial_target"
|
||||
assert partial["reason"] == "本期数据只到 2026-10-20,还差 5 天,和完整的周比没有意义"
|
||||
assert partial["params"]["today"] == "2026-10-21" and partial["probes"][0]["column"] == "order_date"
|
||||
whole = await analyze("decompose", "additive", snaps=[_two("2026-10-05", "2026-10-12")], probe=probe,
|
||||
base="2026-10-05", target="2026-10-12", **ask)
|
||||
assert whole["status"] == "ok" and not [c for c in whole["caveats"] if c["code"] == "comparability"]
|
||||
# 查不到表的范围:只附一句核对不了,不进判定语
|
||||
unknown = await analyze("decompose", "additive", snaps=[_two("2026-10-05", "2026-10-12")],
|
||||
base="2026-10-05", target="2026-10-12", **ask)
|
||||
note = next(c["text"] for c in unknown["caveats"] if c["code"] == "comparability")
|
||||
assert note == "没能核对本期的数据是否到齐(查不到表里这一列的日期范围)" and note not in unknown["phrase"]
|
||||
ratio_rows = [["线上", "2026-10-12", 50, 1000], ["线上", "2026-10-19", 80, 1000], ["线下", "2026-10-12", 20, 1000],
|
||||
["线下", "2026-10-19", 26, 1000]]
|
||||
ratio_snap = snap("Q1", "SELECT channel, date(order_date, '-6 days', 'weekday 1') AS week, SUM(refund_flag) AS "
|
||||
"refunds, COUNT(*) AS orders FROM orders GROUP BY 1, 2",
|
||||
["channel", "week", "refunds", "orders"], ratio_rows)
|
||||
refused = await analyze("decompose", "ratio", snaps=[ratio_snap], probe=probe, query="Q1", by="channel",
|
||||
period="week", base="2026-10-12", target="2026-10-19", numerator="refunds",
|
||||
denominator="orders")
|
||||
assert refused["reason_code"] == "partial_target"
|
||||
|
||||
|
||||
async def test_two_days_a_week_apart_from_a_plain_date_column_are_two_days():
|
||||
# 期次列直接取的日期:昨天(10-20)对上周同一天,不是两周;10-20 当天的数据已经到了
|
||||
probe = fixed_prober({("orders", "order_date"): ("2025-01-01", "2026-10-20")})
|
||||
result = await analyze("decompose", "additive", snaps=[_two("2026-10-13", "2026-10-20", sql=DAY_SQL, col="day")],
|
||||
probe=probe, query="Q2", by=["region"], period="day", base="2026-10-13",
|
||||
target="2026-10-20", measure="gmv")
|
||||
assert result["status"] == "ok" and not [c for c in result["caveats"] if c["code"] == "comparability"]
|
||||
|
||||
@@ -136,6 +136,44 @@ async def test_two_periods_of_the_same_group():
|
||||
async def test_a_group_with_one_value_cannot_be_tested():
|
||||
s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], [["线上", 5], ["线下", 4], ["线下", 6]])
|
||||
result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount")
|
||||
assert result["verdict"]["code"] == "no_evidence" and result["values"]["p"] is None
|
||||
assert result["table"]["rows"][0][-1] == "样本不够,算不了"
|
||||
assert result["status"] == "refused" and result["reason_code"] == "untestable"
|
||||
assert result["reason"] == "线上只有 1 个样本,算不了差异"
|
||||
assert result["phrase"] == "差异检验没有做成:线上只有 1 个样本,算不了差异"
|
||||
|
||||
|
||||
async def test_no_pair_with_a_p_value_is_refused_not_said_to_be_noise():
|
||||
sql = "SELECT region, AVG(amount) AS m, 0 AS sd, COUNT(*) AS n FROM orders GROUP BY 1"
|
||||
means = snap("Q1", sql, ["region", "m", "sd", "n"], [["华东", 120, 0, 1], ["华南", 80, 0, 1]])
|
||||
mean = await analyze("differ", "mean", snaps=[means], query="Q1", by="region", mean="m", sd="sd", n="n",
|
||||
label="客单价")
|
||||
assert mean["reason_code"] == "untestable" and mean["reason"] == "华东只有 1 个样本、华南只有 1 个样本,算不了差异"
|
||||
assert mean["verdict"] is None and mean["settings"] == {"differ_alpha": 0.05, "min_n": 30}
|
||||
rows = snap("Q2", "SELECT region, amount FROM orders", ["region", "amount"], [["华东", 120], ["华南", 80]])
|
||||
one = await analyze("differ", "rows", snaps=[rows], query="Q2", by="region", value="amount")
|
||||
assert one["reason"] == "华东只有 1 个样本、华南只有 1 个样本,算不了差异"
|
||||
zero = await rate([["华东", 0, 0], ["华南", 3, 10]])
|
||||
assert zero["reason_code"] == "untestable" and zero["reason"] == "华东的样本量为 0,算不了差异"
|
||||
|
||||
|
||||
async def test_pairs_without_a_p_value_are_named_and_not_counted():
|
||||
result = await rate([["西南", 80, 1000], ["华东", 40, 1000], ["华北", 0, 0]])
|
||||
rows = {r[0]: r for r in result["table"]["rows"]}
|
||||
assert rows["华北"][-1] == "华北的样本量为 0,算不了" and rows["华北"][10] is None
|
||||
note = next(c for c in result["caveats"] if c["code"] == "untested")
|
||||
assert note["text"] == "另有 1 对算不了(华北的样本量为 0)"
|
||||
# 华北一单都没有:西南的其余合计就是华东,两对都站得住;华北那一对不算进「比了几对」
|
||||
assert result["phrase"] == ("比了 2 对,2 对差异站得住:西南的退款率(8.0%)高于其余各地区(4.0%);华东的退款率(4.0%)"
|
||||
"低于其余各地区(8.0%);另有 1 对算不了(华北的样本量为 0)")
|
||||
assert {"code": "multiple_comparisons", "text": "比了 2 对,已按次数校正"} in result["caveats"]
|
||||
|
||||
|
||||
async def test_the_outlier_is_looked_for_in_the_real_groups_not_in_the_pooled_rest():
|
||||
# 三组各和其余合计比:最远的那一笔在第二组(华南)。只在真实的各组里找、由各组重建「其余」合计,去掉它以后华南高出来就站得住
|
||||
base = [10, 11, 9, 10, 12, 8, 10, 11, 10, 9, 11, 10]
|
||||
rows = ([["华东", v] for v in base] + [["华南", v + 2] for v in base[:-1]] + [["华南", 100]]
|
||||
+ [["华北", v] for v in base])
|
||||
s = snap("Q1", "SELECT region, amount FROM orders", ["region", "amount"], rows)
|
||||
result = await analyze("differ", "rows", snaps=[s], query="Q1", by="region", value="amount", label="客单价")
|
||||
assert result["verdict"]["code"] == "no_evidence"
|
||||
assert {"code": "outlier_sensitive", "text": "去掉最大的一笔后,差异变得站得住"} in result["caveats"]
|
||||
assert math.isfinite(float(result["values"]["diff"]))
|
||||
|
||||
@@ -136,7 +136,7 @@ async def test_the_phrase_opens_judgment_method_calculation_inputs_recompute_and
|
||||
assert judged["cited"] == {"what": "phrase"} and judged["render_ok"] is True and judged["eid_ok"] is True
|
||||
assert judged["origin"] == "card" and judged["item"] == "drop" and judged["label"] == "分析"
|
||||
assert judged["hypothesis"] == "华东拖累" and judged["hypothesis_note"].startswith("Agent 写的假设")
|
||||
assert {c["code"] for c in judged["caveats"]} == {"measure_unlabeled", "headline_basis"}
|
||||
assert {c["code"] for c in judged["caveats"]} == {"measure_unlabeled", "headline_basis", "comparability"}
|
||||
assert method["method"] == "decompose.additive@1" and method["method_label"] == "变化拆解(可加)"
|
||||
assert {s["key"]: s["label"] for s in method["settings"]}["main_share"] == "「主要来源」的占比"
|
||||
assert {s["key"] for s in method["settings"]} >= {"main_share", "concentration", "table_top_k"}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""期次(app/analysis/periods.py):日期、YYYY-MM、YYYY-Www、YYYY-Qn、整数序号认得出、能排序、知道覆盖哪几天;粒度按值推断;
|
||||
写法混着、带时刻的认不出。"""
|
||||
写法混着、带时刻的认不出。周号按 SQL 里的格式串认算法(SQLite 的 %W / %U / %G-%V、MySQL 的 %U / %u / %X-%V / %x-%v、to_char
|
||||
的 WW / IW),按算法算真实起止;认不出算法的起止为空。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
@@ -26,10 +27,10 @@ def test_labels_for_months_weeks_quarters_and_sequences():
|
||||
assert grain == "month" and items[1].ordinal - items[0].ordinal == 1 and items[1].end == date(2026, 10, 31)
|
||||
items, grain = pd.parse([202609, 202610])
|
||||
assert grain == "month"
|
||||
items, grain = pd.parse(["2026-W41", "2026-W42", "2026-W01"])
|
||||
items, grain = pd.parse(["2026-W41", "2026-W42", "2026-W01"], weeks="iso")
|
||||
assert grain == "week" and items[1].start == date(2026, 10, 12) and items[1].end == date(2026, 10, 18)
|
||||
assert items[1].ordinal - items[0].ordinal == 1
|
||||
items, grain = pd.parse(["2025-W52", "2026-W01"]) # 跨年也连着
|
||||
items, grain = pd.parse(["2025-W52", "2026-W01"], weeks="iso") # 跨年也连着
|
||||
assert items[1].ordinal - items[0].ordinal == 1
|
||||
items, grain = pd.parse(["2026Q3", "2026-Q4"])
|
||||
assert grain == "quarter" and items[1].start == date(2026, 10, 1)
|
||||
@@ -55,3 +56,94 @@ def test_calendar_units_and_weekend_days():
|
||||
assert pd.weekend_days(date(2026, 10, 16), date(2026, 10, 18)) == 2 # 周五到周日
|
||||
assert pd.weekend_days(date(2026, 10, 12), date(2026, 10, 14)) == 0
|
||||
assert pd.weekend_days(date(2026, 10, 1), date(2026, 10, 31)) == 9
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 周号:按 SQL 认算法
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
WEEK_SQL = {
|
||||
("sqlite", "SELECT strftime('%Y-W%W', day) AS week, SUM(amount) AS gmv FROM orders GROUP BY week"): "monday",
|
||||
("sqlite", "SELECT strftime('%Y-W%U', day) AS week FROM orders GROUP BY 1"): "sunday",
|
||||
("sqlite", "SELECT strftime('%G-W%V', day) AS week FROM orders GROUP BY 1"): "iso",
|
||||
("sqlite", "SELECT strftime('%Y', day) || '-W' || strftime('%W', day) AS week FROM orders GROUP BY 1"): "monday",
|
||||
("sqlite", "WITH w AS (SELECT strftime('%Y-W%W', day) AS wk, amount FROM orders) "
|
||||
"SELECT wk AS week, SUM(amount) AS gmv FROM w GROUP BY 1"): "monday",
|
||||
("sqlite", "SELECT t.wk AS week FROM (SELECT strftime('%G-W%V', day) AS wk FROM orders) AS t GROUP BY 1"): "iso",
|
||||
("mysql", "SELECT DATE_FORMAT(day, '%x-W%v') AS week FROM orders GROUP BY 1"): "iso",
|
||||
("mysql", "SELECT DATE_FORMAT(day, '%X-W%V') AS week FROM orders GROUP BY 1"): "sunday_year",
|
||||
("mysql", "SELECT DATE_FORMAT(day, '%Y-W%u') AS week FROM orders GROUP BY 1"): "monday4",
|
||||
("mysql", "SELECT DATE_FORMAT(day, '%Y-W%U') AS week FROM orders GROUP BY 1"): "sunday",
|
||||
("postgres", "SELECT to_char(day, 'IYYY-\"W\"IW') AS week FROM orders GROUP BY 1"): "iso",
|
||||
("postgres", "SELECT to_char(day, 'YYYY-\"W\"WW') AS week FROM orders GROUP BY 1"): "jan1",
|
||||
("oracle", "SELECT TO_CHAR(day, 'IYYY-\"W\"IW') AS week FROM orders GROUP BY TO_CHAR(day, 'IYYY-\"W\"IW')"): "iso",
|
||||
("oracle", "SELECT to_char(day, 'YYYY\"W\"WW') AS week FROM orders"): "jan1",
|
||||
}
|
||||
|
||||
|
||||
def test_the_week_algorithm_is_read_from_the_sql():
|
||||
for (dialect, sql), scheme in WEEK_SQL.items():
|
||||
assert pd.week_scheme(sql, "week", dialect) == scheme, (dialect, sql)
|
||||
|
||||
|
||||
def test_week_algorithms_that_cannot_be_told_are_unknown():
|
||||
# 年份和周号不配套:同一个标签会落在两段日子上
|
||||
assert pd.week_scheme("SELECT strftime('%Y-W%V', day) AS week FROM orders", "week", "sqlite") is None
|
||||
assert pd.week_scheme("SELECT DATE_FORMAT(day, '%Y-W%v') AS week FROM orders", "week", "mysql") is None
|
||||
assert pd.week_scheme("SELECT to_char(day, 'YYYY-\"W\"IW') AS week FROM orders", "week", "postgres") is None
|
||||
# MySQL 的 %W 是星期几的名字,不是周号;方言不知道;取的是现成的一列;两个日期列
|
||||
assert pd.week_scheme("SELECT DATE_FORMAT(day, '%Y-W%W') AS week FROM orders", "week", "mysql") is None
|
||||
assert pd.week_scheme("SELECT strftime('%Y-W%W', day) AS week FROM orders", "week", None) is None
|
||||
assert pd.week_scheme("SELECT week, SUM(amount) AS gmv FROM weekly GROUP BY week", "week", "sqlite") is None
|
||||
assert pd.week_scheme("SELECT strftime('%Y', a) || '-W' || strftime('%W', b) AS week FROM t", "week", "sqlite") \
|
||||
is None
|
||||
assert pd.week_scheme("SELECT strftime('%Y-W%W', day) AS wk FROM orders", "week", "sqlite") is None
|
||||
|
||||
|
||||
def test_sqlite_monday_weeks_cut_at_new_year():
|
||||
items, grain = pd.parse(["2025-W52", "2026-W00", "2026-W01", "2026-W42"], weeks="monday")
|
||||
spans = [(i.start, i.end) for i in items]
|
||||
assert spans == [(date(2025, 12, 29), date(2025, 12, 31)), (date(2026, 1, 1), date(2026, 1, 4)),
|
||||
(date(2026, 1, 5), date(2026, 1, 11)), (date(2026, 10, 19), date(2026, 10, 25))]
|
||||
assert [i.ordinal - items[0].ordinal for i in items[:3]] == [0, 1, 2] # 跨年切开的两期也连着
|
||||
assert [pd.short_days(i, grain) for i in items] == [3, 4, None, None]
|
||||
# 1 月 1 日就是周一的年份没有第 0 周:认不出,原因里举的是它
|
||||
assert pd.parse(["2024-W00", "2024-W01"], weeks="monday") is None
|
||||
assert pd.unreadable(["2024-W00", "2024-W01"], weeks="monday") == ("week", ["2024-W00"])
|
||||
# ISO 只有 52 周的年份没有第 53 周
|
||||
assert pd.parse(["2026-W53"], weeks="iso") is not None and pd.parse(["2025-W53"], weeks="iso") is None
|
||||
|
||||
|
||||
def test_weeks_of_an_unknown_algorithm_have_no_dates_but_keep_their_order():
|
||||
items, grain = pd.parse(["2025-W51", "2025-W52", "2026-W00", "2026-W01", "2026-W03"])
|
||||
assert grain == "week" and all(i.start is None and i.end is None for i in items)
|
||||
assert [i.ordinal - items[0].ordinal for i in items] == [0, 1, 2, 3, 5] # 同一年里隔了一周就是缺期
|
||||
items, _ = pd.parse(["2025-W52", "2026-W01"]) # 上一年最后一周接下一年第一周
|
||||
assert items[1].ordinal - items[0].ordinal == 1
|
||||
items, _ = pd.parse(["2025-W40", "2026-W01"])
|
||||
assert items[1].ordinal - items[0].ordinal > 1
|
||||
|
||||
|
||||
def test_unreadable_values_are_named():
|
||||
assert pd.unreadable(["2026-W40", "上周", "2026-W41", None]) == ("shape", ["上周", None])
|
||||
assert pd.unreadable(["2026-W40", "2026-W41", "2026-10-12"]) == ("mixed", ["2026-10-12"])
|
||||
assert pd.unreadable(["2026-10", "2026-11"], "week") == ("grain", ["2026-10", "2026-11"])
|
||||
|
||||
|
||||
def test_a_plain_date_column_is_a_day_even_when_two_values_are_a_week_apart():
|
||||
plain = "SELECT region, order_date AS day, SUM(amount) AS gmv FROM orders GROUP BY 1, 2"
|
||||
starts = ("SELECT region, date(order_date, '-6 days', 'weekday 1') AS week, SUM(amount) AS gmv FROM orders "
|
||||
"GROUP BY 1, 2")
|
||||
assert pd.plain_day(plain, "day", "sqlite") and pd.plain_day(plain.replace("order_date AS", "DATE(order_date) AS"),
|
||||
"day", "sqlite")
|
||||
assert not pd.plain_day(starts, "week", "sqlite")
|
||||
assert pd.parse_compared(["2026-10-13", "2026-10-20"], plain, "day", "sqlite")[1] == "day"
|
||||
assert pd.parse_compared(["2026-10-12", "2026-10-19"], starts, "week", "sqlite")[1] == "week"
|
||||
assert pd.parse_compared(["2026-09-01", "2026-10-01"], plain, "day", "sqlite")[1] == "month"
|
||||
|
||||
|
||||
|
||||
def test_week_numbers_out_of_any_algorithm_are_unreadable():
|
||||
assert pd.parse(["2026-W41", "2026-W99"]) is None
|
||||
assert pd.unreadable(["2026-W41", "2026-W99"]) == ("shape", ["2026-W99"])
|
||||
assert pd.week_span("sunday_year", 9999, 53) is None and pd.week_span("iso", 0, 1) is None
|
||||
|
||||
@@ -60,6 +60,28 @@ def test_labels_are_scrubbed_of_numbers_units_and_markers():
|
||||
assert ph.pick_label("2026", None, "2026") == "这一列"
|
||||
|
||||
|
||||
def test_scrubbing_a_scrubbed_label_changes_nothing():
|
||||
# 去掉一处可能拼出新的一处:去完数字的「销售额[[v:Q.r.gmv]]」又是一个引用标记,要接着去
|
||||
assert ph.clean_label("销售额[ [v:Q1.r0.gmv] ]") == "销售额"
|
||||
assert ph.clean_label("[ [see:A1] ]销售额") == "销售额" and ph.clean_label("销售额[1[v:x]2]") == "销售额"
|
||||
for text in ("销售额[ [v:Q1.r0.gmv] ]", "销售额 [ [ a:A1 ] ]", "销售额( )(3)", "销售额(同比 +30%,3.5万)",
|
||||
"近三成客户的退款率", "【线上】销售额", "退款率 (%)", "销售额(", " ,销售额。", "一[二]三",
|
||||
"销售额" * 30, "[[a:A1]]主要来自[[v:Q1.r0.region]]"):
|
||||
once = ph.clean_label(text)
|
||||
assert ph.clean_label(once) == once, text
|
||||
|
||||
|
||||
def test_labels_stop_at_the_first_clause_and_judgments_fall_back():
|
||||
assert ph.clean_label("销售额,主要来自华东") == "销售额"
|
||||
assert ph.clean_label("订单数; 销售额") == "订单数" and ph.clean_label("订单数、销售额") == "订单数"
|
||||
assert ph.clean_label("销售额(含税,元)") == "销售额" # 截完没配上对的括号也不要
|
||||
# 判定语不受说法核对:叫法里有判断词的整个不要,退回下一个来源(目录叫法、当叫法用的列名同样处理)
|
||||
assert ph.clean_label("销售额主要来自华东") == "" and ph.clean_label("历史新高的销售额") == ""
|
||||
assert ph.pick_label("销售额主要来自华东", "实收金额", "gmv") == "实收金额"
|
||||
assert ph.pick_label(None, "历史新高的销售额", "gmv") == "gmv"
|
||||
assert ph.pick_label(None, None, "主要来自华东") == "这一列"
|
||||
|
||||
|
||||
def test_numbers_stuffed_into_a_label_never_reach_the_phrase():
|
||||
label = ph.pick_label("销售额(同比 +30%,3.5万)", "销售额", "gmv")
|
||||
phrase = ph.decompose_driver(label, ph.change(-194, 2220), "华东", "-120", Fraction(6186, 10000))
|
||||
@@ -119,3 +141,18 @@ def test_criteria_are_written_with_the_actual_thresholds():
|
||||
assert "3 倍以上算明显超出" in ph.criterion_baseline(DEFAULTS, 12, "周")
|
||||
assert "小于 0.05" in ph.criterion_trend(DEFAULTS, 10, "周")
|
||||
assert "Fisher" in ph.criterion_differ(DEFAULTS, "rate") and "Welch" in ph.criterion_differ(DEFAULTS, "mean")
|
||||
|
||||
|
||||
|
||||
def test_the_run_at_the_end_of_a_trend_says_whether_it_goes_the_same_way():
|
||||
assert ph.trend_phrase("falling", "销售额", 10, "周", "周", "-2.0%", (3, "down")) == \
|
||||
"过去 10 周销售额呈下降趋势(每周约 -2.0%),已连续 3 周下降"
|
||||
assert ph.trend_phrase("falling", "销售额", 10, "周", "周", "-2.0%", (3, "up")) == \
|
||||
"过去 10 周销售额呈下降趋势(每周约 -2.0%),但最近已连续 3 周上升"
|
||||
assert ph.trend_phrase("rising", "订单数", 8, "个月", "月", None, (4, "down")) == \
|
||||
"过去 8 个月订单数呈上升趋势,但最近已连续 4 个月下降"
|
||||
|
||||
|
||||
def test_simpson_says_the_shift_only_when_the_phrase_does_not():
|
||||
assert ph.simpson_text("退款率", True, "线上", 0.3, 0.5) == "各段的退款率大多在下降,整体却上升:变化来自结构,线上占比从 30.0% 升到 50.0%"
|
||||
assert ph.simpson_text("退款率", True, "线上", 0.3, 0.5, shift=False) == "各段的退款率大多在下降,整体却上升"
|
||||
|
||||
@@ -153,3 +153,109 @@ async def test_ledger_entries_name_the_real_input_snapshots():
|
||||
def test_the_call_limit_error_is_spoken_with_the_limit():
|
||||
error = analysis.exhausted({"max_calls": 5})
|
||||
assert error.code == "max_calls" and error.message == "这个 Agent 这次执行最多做 5 次分析,已经用完:用已有的分析结论写,不再新做分析"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 随数据变化的调用错误(分析卡传 refuse_data_errors):记成算不了的结论,复算也成立
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
ONE_REGION = snap("Q6", WEEKLY_SQL, ["region", "week", "gmv"], [["华东", "2026-W41", 820], ["华东", "2026-W42", 700]])
|
||||
TOTALS = snap("Q7", "SELECT week, SUM(amount) AS gmv FROM orders GROUP BY week", ["week", "gmv"],
|
||||
[["2026-W42", 1926], ["2026-W41", None]])
|
||||
FACTORS = snap("Q8", "SELECT week, SUM(amount) AS gmv, COUNT(*) AS orders, AVG(amount) AS aov FROM orders "
|
||||
"GROUP BY week",
|
||||
["week", "gmv", "orders", "aov"], [["2026-W41", 2120, 20, 106], ["2026-W42", None, 0, None]])
|
||||
ONE_GROUP = snap("Q9", "SELECT region, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY region",
|
||||
["region", "refunds", "orders"], [["西南", 80, 1000]])
|
||||
WEEK_RATES = snap("Q10", "SELECT week, SUM(refund_flag) AS refunds, COUNT(*) AS orders FROM orders GROUP BY week",
|
||||
["week", "refunds", "orders"], [["2026-W41", 30, 1000], ["2026-W42", None, 1000]])
|
||||
REFUSALS = [WEEKLY, SERIES, RATES, ONE_REGION, TOTALS, FACTORS, ONE_GROUP, WEEK_RATES]
|
||||
BY_REGION = {"query": "Q2", "by": ["region"], "period": "week", "base": "2026-W41", "target": "2026-W42",
|
||||
"measure": "gmv"}
|
||||
|
||||
|
||||
async def _refused(op, mode=None, *, parents=None, **params):
|
||||
"""refuse_data_errors 为真跑两次:逐字节相同、形状照契约、复算逐字段相同(analyze 里按真复算);不传时照旧是调用错误。"""
|
||||
one = await analyze(op, mode, snaps=REFUSALS, parents=parents, refuse_data_errors=True, **params)
|
||||
two = await analyze(op, mode, snaps=REFUSALS, parents=parents, refuse_data_errors=True, **params)
|
||||
assert json.dumps(one, sort_keys=True, ensure_ascii=False) == json.dumps(two, sort_keys=True, ensure_ascii=False)
|
||||
_assert_shape(one)
|
||||
assert one["status"] == "refused" and one["phrase"] == f"{c.OP_LABELS[op]}没有做成:{one['reason']}"
|
||||
error = await call_error(op, mode, snaps=REFUSALS, parents=parents, **params)
|
||||
assert error.code in analysis.DATA_ERRORS and error.reason == one["reason"]
|
||||
return one
|
||||
|
||||
|
||||
def test_the_data_errors_are_exported():
|
||||
assert analysis.DATA_ERRORS == {"parent_refused", "parent_key_missing", "period_not_found", "row_out_of_range",
|
||||
"headline_empty", "unknown_group", "too_few_groups", "empty_cell"}
|
||||
assert {"parent_refused", "input_unavailable"} <= set(c.REASONS)
|
||||
|
||||
|
||||
async def test_a_drill_under_a_refused_parent_is_itself_refused():
|
||||
parent = await analyze("decompose", "additive", snaps=REFUSALS, **{**BY_REGION, "query": "Q6"})
|
||||
assert parent["reason_code"] == "too_few_segments"
|
||||
result = await _refused("decompose", "additive", parents={"where": parent},
|
||||
**{**BY_REGION, "by": ["region"], "parent": {"of": "where", "key": ["华东"]}})
|
||||
assert result["reason_code"] == "parent_refused"
|
||||
assert result["reason"] == "上一层「where」没有做成(按地区只有 1 段,至少要 2 段才拆得开),不能在它下面下钻"
|
||||
# 上一层、取过的快照都记进输入(复算按它们走到同一处);参数照调用原样,阈值同这个模式自己算不了时
|
||||
assert [i["role"] for i in result["inputs"]] == ["parent", "data"]
|
||||
assert result["inputs"][1] == {"ref": "Q2", "artifact": WEEKLY["artifact"], "role": "data",
|
||||
"columns": ["region", "gmv", "week"], "rows": "all"}
|
||||
assert result["params"]["parent"] == {"of": "where", "key": ["华东"]}
|
||||
assert result["settings"] == {**{k: c.DEFAULTS[k] for k in ("main_share", "concentration", "offset_ratio",
|
||||
"flat_pct", "table_top_k")}, "max_drill_depth": 3}
|
||||
assert result["table"] is None and result["verdict"] is None
|
||||
|
||||
|
||||
async def test_segments_periods_cells_and_groups_missing_this_time():
|
||||
parent = await analyze("decompose", "additive", snaps=REFUSALS, **BY_REGION)
|
||||
key = await _refused("decompose", "additive", parents={"A1": parent},
|
||||
**{**BY_REGION, "parent": {"of": "A1", "key": ["东北"]}})
|
||||
assert key["reason_code"] == "input_unavailable" and key["reason"] == "上一层「A1」里没有「东北」这一段"
|
||||
period = await _refused("decompose", "additive", **{**BY_REGION, "target": "2026-W43", "label": "销售额,主要来自华东"})
|
||||
assert period["reason"] == "期次列「week」里没有本期「2026-W43」" and period["params"]["label"] == "销售额"
|
||||
row = await _refused("decompose", "additive", **BY_REGION, headline={"base": "Q7.r5.gmv", "target": "Q7.r0.gmv"})
|
||||
assert row["reason"] == "基期总数那一格(Q7.r5.gmv)不在:Q7 这次只有 2 行"
|
||||
assert [i["role"] for i in row["inputs"]] == ["data", "headline_base"]
|
||||
empty = await _refused("decompose", "additive", **BY_REGION, headline={"base": "Q7.r1.gmv", "target": "Q7.r0.gmv"})
|
||||
assert empty["reason"] == "基期总数那一格(Q7.r1.gmv)是空的"
|
||||
factor = await _refused("decompose", "factor", query="Q8", period="week", base="2026-W41", target="2026-W42",
|
||||
product="gmv", factors=["orders", "aov"])
|
||||
assert factor["reason"] == "本期的「gmv」那一格是空的(Q8 第 1 行)"
|
||||
target = await _refused("baseline", query="Q1", period="week", measure="gmv", target="2020-01-06")
|
||||
assert target["reason"] == "期次列「week」里没有本期「2020-01-06」(或那一期为空)"
|
||||
assert target["settings"] == {k: c.DEFAULTS[k] for k in ("k_warn", "k_alert", "min_history", "max_history")}
|
||||
side = await _refused("comparable", query="Q2", period="week", base="2026-W40", target="2026-W42")
|
||||
assert side["reason"] == "期次列「week」里没有基期「2026-W40」" and side["settings"] == {}
|
||||
rates = {"by": ["region"], "numerator": "refunds", "denominator": "orders"}
|
||||
group = await _refused("differ", "rate", query="Q3", **rates, pairs=[["华东", "东北"]])
|
||||
assert group["reason"] == "分组列「region」里没有「东北」这一组"
|
||||
assert group["settings"] == {"differ_alpha": 0.05, "min_expected": 5}
|
||||
few = await _refused("differ", "rate", query="Q9", **rates)
|
||||
assert few["reason"] == "分组列「region」里只有 1 组,至少要两组才比得了"
|
||||
weeks = {"query": "Q10", "period": "week", "numerator": "refunds", "denominator": "orders"}
|
||||
gone = await _refused("differ", "rate", **weeks, base="2026-W40", target="2026-W41")
|
||||
assert gone["reason"] == "期次列「week」里没有基期「2026-W40」"
|
||||
null = await _refused("differ", "rate", **weeks, base="2026-W41", target="2026-W42")
|
||||
assert null["reason"] == "「refunds」那一格是空的(Q10 第 1 行)"
|
||||
|
||||
|
||||
async def test_structural_call_errors_still_fail_even_when_data_errors_are_refused():
|
||||
for params in ({**BY_REGION, "by": ["channel"]}, {**BY_REGION, "by": ["region", "week"]}):
|
||||
with pytest.raises(CallError) as e:
|
||||
await analyze("decompose", "additive", snaps=REFUSALS, refuse_data_errors=True, check=False, **params)
|
||||
assert e.value.code not in analysis.DATA_ERRORS and e.value.reason is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("op,mode,params", [CALLS[0], *CALLS[2:]])
|
||||
async def test_the_fake_results_for_tests_have_each_operators_shape(op, mode, params):
|
||||
from app.analysis.testing import fake_result
|
||||
|
||||
real = await _run(op, mode, params)
|
||||
fake = fake_result(op, mode, real["verdict"]["code"])
|
||||
assert fake["method"] == real["method"] and fake["table"]["columns"] == real["table"]["columns"]
|
||||
assert set(fake["table"]["labels"]) == set(fake["table"]["columns"])
|
||||
assert set(fake["values"]) <= set(real["values"]) and {i["role"] for i in fake["inputs"]} == \
|
||||
{i["role"] for i in real["inputs"]}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
lag 7、新高新低;上升 / 下降 / 无趋势、连续期数、缺期。每个用例都顺带核对复算和运行逐字段相同。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import date, timedelta
|
||||
from fractions import Fraction
|
||||
|
||||
@@ -210,3 +211,119 @@ async def test_daily_trends_warn_about_weekday_effects():
|
||||
assert any(c["code"] == "weekday_effect" and c["text"] == "日数据有星期效应,看趋势建议按周汇总"
|
||||
for c in result["caveats"])
|
||||
assert "日数据有星期效应" in result["phrase"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 周号按 SQL 的算法算起止(SQLite 的 strftime('%W') 不是 ISO 周)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: SQLite 按 %W 写的周:第 1 周从这一年第一个周一起,之前的几天是第 0 周,跨年那一周被切成两期
|
||||
WEEKNO = "SELECT strftime('%Y-W%W', order_date) AS week, COUNT(*) AS orders FROM orders GROUP BY week ORDER BY week"
|
||||
|
||||
|
||||
def sqlite_weeks(first, last, sql=WEEKNO):
|
||||
"""真用 SQLite 按 sql 汇总:每天 100 笔订单,从 first 到 last(含)。"""
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE orders (order_date TEXT)")
|
||||
day = first
|
||||
while day <= last:
|
||||
db.executemany("INSERT INTO orders VALUES (?)", [(day.isoformat(),)] * 100)
|
||||
day += timedelta(days=1)
|
||||
rows = [list(r) for r in db.execute(sql)]
|
||||
db.close()
|
||||
return rows
|
||||
|
||||
|
||||
def weekly(rows, sql=WEEKNO, **kw):
|
||||
return snap("Q1", sql, ["week", "orders"], rows, types={"week": "text", "orders": "number"}, **kw)
|
||||
|
||||
|
||||
async def test_a_sqlite_week_that_is_not_over_is_not_a_whole_week():
|
||||
# 运行当天 10-21、数据到 10-20:SQLite 的 2026-W42 是 10-19 到 10-25,只有两天的数据(ISO 的 W42 是 10-12 到 10-18)
|
||||
rows = sqlite_weeks(date(2026, 6, 1), date(2026, 10, 20))
|
||||
assert rows[-1] == ["2026-W42", 200]
|
||||
probe = fixed_prober({("orders", "order_date"): ("2026-06-01", "2026-10-20")})
|
||||
result = await analyze("baseline", snaps=[weekly(rows)], probe=probe, query="Q1", period="week", measure="orders",
|
||||
label="订单数")
|
||||
assert result["reason_code"] == "partial_target"
|
||||
assert result["reason"] == "本期数据只到 2026-10-20,还差 5 天,和完整的周比没有意义"
|
||||
assert result["params"]["target"] == "2026-W42"
|
||||
whole = await analyze("baseline", snaps=[weekly(rows)], probe=probe, query="Q1", period="week", measure="orders",
|
||||
label="订单数", target="2026-W41")
|
||||
assert whole["status"] == "ok" and whole["verdict"]["code"] == "within_band"
|
||||
# 不知道方言(认不出算法):起止为空,本期齐不齐核对不了,绝不判齐
|
||||
unknown = await analyze("baseline", snaps=[weekly(rows)], query="Q1", period="week", measure="orders",
|
||||
label="订单数", target="2026-W41")
|
||||
note = next(c["text"] for c in unknown["caveats"] if c["code"] == "comparability")
|
||||
assert note == "没能核对本期的数据是否到齐(期次写成「年-W周号」,认不出 SQL 是按哪种周号算的,不知道是哪几天)"
|
||||
|
||||
|
||||
async def test_sqlite_weeks_across_new_year_are_readable_and_the_cut_weeks_are_left_out():
|
||||
rows = sqlite_weeks(date(2025, 9, 29), date(2026, 3, 1))
|
||||
assert ["2025-W52", 300] in rows and ["2026-W00", 400] in rows
|
||||
probe = fixed_prober({("orders", "order_date"): ("2025-09-29", "2026-03-01")})
|
||||
ask = {"snaps": [weekly(rows)], "probe": probe, "query": "Q1", "period": "week", "measure": "orders",
|
||||
"label": "订单数"}
|
||||
trend = await analyze("trend", **ask)
|
||||
# 最后连着的整周是 2026-W01 到 W08:再往前的 2026-W00 只有 4 天
|
||||
assert trend["status"] == "ok" and trend["values"]["n"] == "8" and trend["verdict"]["code"] == "no_trend"
|
||||
assert [r[0] for r in trend["table"]["rows"]][0] == "2026-W01"
|
||||
year_end = await analyze("baseline", **ask, target="2026-W00")
|
||||
assert year_end["reason_code"] == "partial_target"
|
||||
assert year_end["reason"] == "本期只有 2026-01-01 到 2026-01-04 这 4 天,不是整周,和完整的周比没有意义"
|
||||
after = await analyze("baseline", **ask, target="2026-W01")
|
||||
assert after["reason_code"] == "target_gap"
|
||||
assert after["reason"] == "比较期(2026-W00)只有 4 天,不是整周,和它比没有意义"
|
||||
later = await analyze("baseline", **ask, target="2026-W03")
|
||||
# 历史:2025-W40 到 W51 各和上一周比(12 个),2026-W02 和 W01 比(1 个);W01 和 W00、W00 和 2025-W52 都不算
|
||||
assert later["status"] == "ok" and later["values"]["n_history"] == "13"
|
||||
assert {"code": "partial_periods", "text": "有 2 期不到整周(跨年被切开),没有算进历史"} in later["caveats"]
|
||||
assert "跨年" not in later["phrase"]
|
||||
short = await analyze("trend", **{**ask, "snaps": [weekly([r for r in rows if r[0] <= "2026-W03"])]})
|
||||
assert short["reason_code"] == "series_gap"
|
||||
assert short["reason"] == "最后连着的只有 3 期,再往前的 2026-W00 只有 4 天,不是整周(至少要连着的 6 期)"
|
||||
|
||||
|
||||
async def test_unreadable_periods_name_the_values_that_cannot_be_read():
|
||||
rows = [[f"2026-W{w:02d}", 100] for w in range(30, 42)] + [["上周", 100]]
|
||||
result = await baseline(rows)
|
||||
assert result["reason_code"] == "period_unreadable"
|
||||
assert result["reason"] == "期次列「week」的值认不出是哪一期(上周):要日期、YYYY-MM、YYYY-Www、YYYY-Qn 或整数序号"
|
||||
mixed = await baseline([[f"2026-W{w:02d}", 100] for w in range(30, 42)] + [["2026-10-12", 100]])
|
||||
assert mixed["reason"] == "期次列「week」的写法不一:2026-10-12 和其余的不是同一种写法"
|
||||
missing = await analyze("baseline", snaps=[weekly([["2024-W00", 1], ["2024-W01", 2]])], probe=fixed_prober(),
|
||||
query="Q1", period="week", measure="orders")
|
||||
assert missing["reason"] == "期次列「week」里的 2024-W00 按 SQL 里的周号算法没有这一周"
|
||||
|
||||
|
||||
async def test_an_empty_comparison_period_is_said_as_empty_not_as_a_gap():
|
||||
rows = weeks(NOISE + [100, 101])
|
||||
rows[-2][1] = None # 本期的上一周有这一行,值为空
|
||||
result = await baseline(rows)
|
||||
assert result["reason_code"] == "target_gap"
|
||||
assert result["reason"] == f"比较期({rows[-2][0]})的订单数为空,没有比较期的值"
|
||||
tail = weeks([100 + i for i in range(10)])
|
||||
tail[-4][1] = None
|
||||
broken = await trend(tail)
|
||||
assert broken["reason"] == f"最后连着的只有 3 期,再往前的 {tail[-4][0]} 那一期的线上销售额为空(至少要连着的 6 期)"
|
||||
|
||||
|
||||
async def test_sequence_periods_say_completeness_cannot_be_checked():
|
||||
rows = [[i + 1, v] for i, v in enumerate([10, 12, 11, 13, 9, 10, 12, 11, 16])]
|
||||
result = await baseline(rows, basis="level")
|
||||
assert {"code": "comparability", "text": "没能核对本期的数据是否到齐(期次不是日期,不知道是哪几天)"} \
|
||||
in result["caveats"]
|
||||
|
||||
|
||||
async def test_a_falling_trend_whose_last_weeks_rise_says_but():
|
||||
falling = await trend(weeks([200, 196, 190, 188, 181, 176, 170, 165, 160, 162, 165, 168]))
|
||||
assert falling["verdict"]["code"] == "falling" and falling["values"]["run_direction"] == "up"
|
||||
assert falling["phrase"].endswith(",但最近已连续 3 周上升")
|
||||
|
||||
|
||||
async def test_a_period_column_named_like_a_result_column_is_a_call_error():
|
||||
rows = [[r[0], r[1]] for r in weeks(NOISE + [101])]
|
||||
named = snap("Q1", "SELECT DATE(order_date, 'weekday 0', '-6 days') AS value, COUNT(*) AS orders FROM orders "
|
||||
"GROUP BY 1", ["value", "orders"], rows)
|
||||
error = await call_error("baseline", snaps=[named], query="Q1", period="value", measure="orders")
|
||||
assert error.code == "reserved_column" and "期次列「value」和结果表的列重名" in error.message
|
||||
|
||||
@@ -69,6 +69,7 @@ def test_the_group_is_the_contract():
|
||||
broken = {"main_share": "一半", "max_calls": 3.5, "k_warn": 4, "k_alert": 3.5, "min_history": 20, "max_history": 10,
|
||||
"table_top_k": 5, "trend_alpha": True}
|
||||
assert analysis_values(broken) == {**contract.DEFAULTS, "table_top_k": 5}
|
||||
assert analysis_values({"main_share": 0.9, "concentration": 0.5}) == contract.DEFAULTS
|
||||
# 浮点的项存成浮点:发起时的快照、结论里记的阈值一个写法
|
||||
assert analysis_values({"k_warn": 3, "k_alert": 4})["k_warn"] == 3.0
|
||||
assert isinstance(analysis_values({"k_warn": 3, "k_alert": 4})["k_alert"], float)
|
||||
@@ -81,6 +82,8 @@ def test_the_group_is_the_contract():
|
||||
({"max_calls": 2.5}, f"{contract.LABELS['max_calls']}要填整数"),
|
||||
({"k_warn": 3.5}, f"{contract.LABELS['k_alert']}要大于{contract.LABELS['k_warn']}"),
|
||||
({"min_history": 60}, f"{contract.LABELS['max_history']}不能小于{contract.LABELS['min_history']}"),
|
||||
({"main_share": 0.9, "concentration": 0.5},
|
||||
f"{contract.LABELS['concentration']}不能小于{contract.LABELS['main_share']}"),
|
||||
])
|
||||
def test_problems_are_said_with_labels(value, why):
|
||||
assert why in (analysis_problem(value) or "")
|
||||
|
||||
Reference in New Issue
Block a user