From 7e6d52004c4298ac5009c530856b31b6480beb50 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 13:03:35 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=86=E6=9E=90=E7=AE=97=E5=AD=90=E7=9A=84?= =?UTF-8?q?=E8=BE=93=E5=85=A5=E3=80=81=E7=9B=AE=E5=BD=95=E4=BA=8B=E5=AE=9E?= =?UTF-8?q?=E3=80=81=E6=9C=9F=E6=AC=A1=E4=B8=8E=E7=8E=B0=E6=9F=A5=EF=BC=9A?= =?UTF-8?q?inputs.py=EF=BC=88=E6=8C=89=E5=90=8D=E5=AD=97=E5=8F=96=E5=BF=AB?= =?UTF-8?q?=E7=85=A7=E3=80=81=E6=8B=86=E3=80=8C=E5=90=8D=E5=AD=97.r?= =?UTF-8?q?=E8=A1=8C.=E5=88=97=E3=80=8D=E3=80=81=E5=88=97=E5=90=8D?= =?UTF-8?q?=E4=B8=8D=E5=88=86=E5=A4=A7=E5=B0=8F=E5=86=99=E3=80=81=E9=81=AE?= =?UTF-8?q?=E7=BD=A9=E5=88=97=E4=B8=8E=E9=9D=9E=E6=95=B0=E5=80=BC=E6=98=AF?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E9=94=99=E8=AF=AF=EF=BC=89=EF=BC=9Bfacts.py?= =?UTF-8?q?=EF=BC=88=E6=8C=89=20SQL=20=E8=AE=A4=E7=BB=93=E6=9E=9C=E5=88=97?= =?UTF-8?q?=E5=8F=96=E8=87=AA=E5=93=AA=E5=BC=A0=E8=A1=A8=E7=9A=84=E5=93=AA?= =?UTF-8?q?=E4=B8=80=E5=88=97=EF=BC=8C=E8=AF=BB=E7=9B=AE=E5=BD=95=E7=9A=84?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E7=B1=BB=E5=9E=8B=E3=80=81=E5=8F=AB=E6=B3=95?= =?UTF-8?q?=E3=80=81=E4=B8=9A=E5=8A=A1=E6=97=A5=E6=9C=9F=EF=BC=8C=E5=8F=AA?= =?UTF-8?q?=E6=9C=89=E7=A1=AE=E8=AE=A4=E8=BF=87=E7=9A=84=E5=BA=A6=E9=87=8F?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E6=89=8D=E6=8B=BF=E6=9D=A5=E5=88=A4=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E9=94=99=E8=AF=AF=EF=BC=9B=E6=9F=A5=E8=BF=87=E7=9A=84?= =?UTF-8?q?=E8=AE=B0=E8=BF=9B=20params=EF=BC=8C=E5=A4=8D=E7=AE=97=E4=B8=8D?= =?UTF-8?q?=E8=AF=BB=E7=9B=AE=E5=BD=95=EF=BC=89=EF=BC=9Bperiods.py?= =?UTF-8?q?=EF=BC=88=E6=97=A5=E6=9C=9F=E3=80=81YYYY-MM=E3=80=81YYYY-Www?= =?UTF-8?q?=E3=80=81YYYY-Qn=E3=80=81=E6=95=B4=E6=95=B0=E5=BA=8F=E5=8F=B7?= =?UTF-8?q?=EF=BC=8C=E7=B2=92=E5=BA=A6=E6=8C=89=E5=80=BC=E6=8E=A8=E6=96=AD?= =?UTF-8?q?=EF=BC=89=EF=BC=9Bprobe.py=EF=BC=88make=5Fprober=20=E8=B5=B0=20?= =?UTF-8?q?data/coverage=20=E7=9A=84=E5=8F=AA=E8=AF=BB=E6=8E=A2=E6=9F=A5?= =?UTF-8?q?=EF=BC=9A=E9=81=AE=E7=BD=A9=E5=88=97=E4=B8=8D=E6=8E=A2=E3=80=81?= =?UTF-8?q?=E6=80=BB=E6=97=B6=E9=99=90=E3=80=815=20=E5=88=86=E9=92=9F?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E3=80=81=E9=80=80=E5=9B=9E=E5=89=96=E6=9E=90?= =?UTF-8?q?=E4=BA=8B=E5=AE=9E=EF=BC=8C=E5=8F=A6=E5=9C=A8=E5=BA=93=E9=87=8C?= =?UTF-8?q?=E7=AE=97=E6=8C=89=E8=BF=90=E8=A1=8C=E5=BD=93=E5=A4=A9=E7=8E=B0?= =?UTF-8?q?=E7=AE=97=E7=9A=84=E7=95=8C=EF=BC=9BBook=20=E5=85=88=E7=99=BB?= =?UTF-8?q?=E8=AE=B0=E3=80=81=E6=9F=A5=E5=AE=8C=E5=86=8D=E7=AE=97=EF=BC=8C?= =?UTF-8?q?=E7=BB=93=E8=AE=BA=E5=8F=AA=E8=AE=B0=E7=9C=9F=E7=94=A8=E5=88=B0?= =?UTF-8?q?=E7=9A=84=EF=BC=89=EF=BC=9Bdata/coverage.py=20=E5=8A=A0?= =?UTF-8?q?=E5=87=A0=E4=B8=AA=E5=85=AC=E5=BC=80=E7=9A=84=E5=B0=8F=E5=87=BD?= =?UTF-8?q?=E6=95=B0=EF=BC=88parse=5Fsql=E3=80=81as=5Fday=E3=80=81literal?= =?UTF-8?q?=5Fbound=E3=80=81window=5Fof=E3=80=81column=5Frange=E3=80=81bou?= =?UTF-8?q?nd=5Fvalues=E3=80=81facts=5Frange=EF=BC=89=EF=BC=8C=5Ffallback?= =?UTF-8?q?=20=E7=9A=84=E8=AF=BB=E6=B3=95=E6=8A=BD=E6=88=90=20=5Ffacts=5Fr?= =?UTF-8?q?ange=E3=80=81=E8=A1=8C=E4=B8=BA=E4=B8=8D=E5=8F=98=EF=BC=9B?= =?UTF-8?q?=E5=A5=91=E7=BA=A6=E8=A1=A5=E5=87=A0=E9=A1=B9=E5=8F=AF=E9=80=89?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=EF=BC=88InputSnapshot=20=E7=9A=84=20dialect?= =?UTF-8?q?=20/=20checks=20/=20no=5Fdata=20/=20mask=5Fcolumns=20/=20masked?= =?UTF-8?q?=5Fat=5Fquery=EF=BC=8CAnalysisInput=20=E7=9A=84=20dialect?= =?UTF-8?q?=EF=BC=8CProbe=20=E7=9A=84=20expr=20/=20value=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/contract.py | 13 +- backend/app/analysis/facts.py | 218 +++++++++++++++++++++ backend/app/analysis/inputs.py | 220 +++++++++++++++++++++ backend/app/analysis/periods.py | 224 ++++++++++++++++++++++ backend/app/analysis/probe.py | 254 +++++++++++++++++++++++++ backend/app/data/coverage.py | 94 ++++++++- backend/tests/test_analysis_periods.py | 57 ++++++ 7 files changed, 1076 insertions(+), 4 deletions(-) create mode 100644 backend/app/analysis/facts.py create mode 100644 backend/app/analysis/inputs.py create mode 100644 backend/app/analysis/periods.py create mode 100644 backend/app/analysis/probe.py create mode 100644 backend/tests/test_analysis_periods.py diff --git a/backend/app/analysis/contract.py b/backend/app/analysis/contract.py index e70d4cf8..8e82268d 100644 --- a/backend/app/analysis/contract.py +++ b/backend/app/analysis/contract.py @@ -188,6 +188,12 @@ class InputSnapshot(TypedDict, total=False): rows: list[list[Any]] truncated: bool masked: list[str] # 不能当维度、度量的列(遮罩列、按 SQL 认出改了名字的遮罩列),比较时不分大小写 + # 以下几项照查询快照里的原样带上(第 1 波 W1-a 加,都可选): + dialect: str # SQL 的 sqlglot 方言(sqlite / mysql / postgres / oracle);不给时按探查器的 dialect(数据源名) + checks: list[dict[str, Any]] # 快照里的 SQL 检查结果(level 为 error 的算可比性的数据问题) + no_data: list[dict[str, Any]] # 快照里的「查询区间里这张表没有数据」 + mask_columns: list[str] # 没给 masked 时(复算取回的是快照工件本身)按这两项认遮罩列 + masked_at_query: list[str] class AnalysisInput(TypedDict, total=False): @@ -201,6 +207,7 @@ class AnalysisInput(TypedDict, total=False): row: int # 一格时:第几行、哪一列、值(十进制字符串) column: str value: str | None + dialect: str # 读这份快照的 SQL 用的方言(第 1 波 W1-a 加,可选):复算按它读,界的写法才一样 class Probe(TypedDict, total=False): @@ -213,6 +220,10 @@ class Probe(TypedDict, total=False): max: str | None basis: Literal["live", "profile", "observed"] at: str # 现查的时刻(ISO,服务端时区) + # 按运行当天现算的界(CURRENT_DATE、date('now', …))在库里算出的值(第 1 波 W1-a 加,可选):有 expr 的这一条记的是界, + # 没有 table / column / min / max + expr: str # 界的 SQL 写法(按这份快照的方言) + value: str | None # 库里算出的日期(带时刻的写成 YYYY-MM-DD HH:MM:SS) class Verdict(TypedDict, total=False): @@ -284,7 +295,7 @@ class Context(TypedDict, total=False): today: date # 运行当天(服务端时区):判本期完不完整 probe: Prober | None # None 时只用 facts catalogs: Mapping[str, Any] # 数据源名 → 数据目录(app/data/catalog 的 notes):认度量类型、业务日期列、列的叫法 - facts: Mapping[str, Any] # 数据源名 → 剖析事实(探查失败时兜底) + facts: Mapping[str, Any] # 数据源名 → 剖析事实(探查失败时兜底):{表名: last_facts 的那一项},或取它的函数 # -------------------------------------------------------------------------- diff --git a/backend/app/analysis/facts.py b/backend/app/analysis/facts.py new file mode 100644 index 00000000..c13d70d7 --- /dev/null +++ b/backend/app/analysis/facts.py @@ -0,0 +1,218 @@ +"""目录事实:结果列给人看的叫法、度量类型,表的业务日期(细案 01 §2.1、§6、§7.3)。 + +算子要知道「gmv 是流量还是比率」(比率当可加是调用错误;窗口不等时流量判不可比、其余只提醒)、「region 叫地区」(判定语 +写「分散在各地区」)、「orders 的业务日期是 order_date」(可比性检查按它取时间条件)。这些在数据目录里,可目录描述的是表的 +列,快照里的是 SQL 的结果列,所以先按 SQL 认结果列取自哪张表的哪一列,认法同单元格的显示规则(engine/cell_format:最外层 +SELECT 里按别名或列名找到那一项,看它是不是聚合、聚合的参数是不是一列、这一列在 FROM / JOIN 的哪张实表上),再查那一列 +的目录项: + +- SUM(x)、不聚合的 x:x 的度量类型;COUNT(*) / COUNT(x):计数,按流量算;AVG(x):平均值,按比率算(不能按段相加); + COUNT(DISTINCT x)、MIN / MAX、算式:认不出(当没标)。 +- 叫法:x 的中文名(目录里的 label);没有就是没有,调用方用列名。 +- 只认没被驳回的项(catalog.visible_notes)。度量类型另记有没有人确认过(confirmed / verified):只有确认过的才拿来判 + 调用错误,推断的只写进附带说明(同 SQL 检查「宁可漏报,不可误报」)。 + +**复算不读目录**:run 时查过的都记进结论的 params["catalog"](record),复算时 Facts 从那里读,同一个键得到同一个答案; +目录事后改了,封存的结论照样复算得出来。 +""" +from __future__ import annotations + +from collections.abc import Mapping +from functools import lru_cache +from typing import Any + +from app.data.names import name_key + +#: 有确证的项状态:只有它们拿来判调用错误 +_STRONG = frozenset({"confirmed", "verified"}) + + +def _notes_of(entry: Any) -> Mapping[str, Any] | None: + """目录里一张表的 notes:CatalogEntry(有 .notes)或 notes 本身;去掉被驳回的项。""" + from app.data import catalog + + notes = getattr(entry, "notes", entry) + return catalog.visible_notes(notes) if isinstance(notes, Mapping) else None + + +def _table_notes(catalog: Mapping[str, Any] | None, table: str | None) -> Mapping[str, Any] | None: + if not catalog or not table: + return None + wanted = {name_key(table), name_key(table.rsplit(".", 1)[-1])} + for name, entry in catalog.items(): + if name_key(str(name)) in wanted or name_key(str(name).rsplit(".", 1)[-1]) in wanted: + return _notes_of(entry) + return None + + +def _item(notes: Mapping[str, Any] | None, column: str, key: str) -> Mapping[str, Any] | None: + columns = (notes or {}).get("columns") if isinstance(notes, Mapping) else None + if not isinstance(columns, Mapping): + return None + for name, items in columns.items(): + if name_key(str(name)) == name_key(column) and isinstance(items, Mapping): + item = items.get(key) + return item if isinstance(item, Mapping) else None + return None + + +@lru_cache(maxsize=128) +def _outer_select(sql: str) -> Any: + """最外层的 SELECT(engine/cell_format 的读法);同一条 SQL 只解析一次(只读,不改这棵树)。""" + from app.engine import cell_format as cf + + try: + return cf._outer_select(sql) + except Exception: # noqa: BLE001 + return None + + +def source_of(sql: str, column: str) -> tuple[str | None, str | None, str | None, bool] | None: + """结果列 column 取自哪里:(表, 列, 聚合, 去重)。聚合是 sum / count / avg / min / max / other(别的聚合)或 None(不聚合); + 表在 SELECT * 、子查询、认不准的时候为 None。SQL 解析不了、结果列在最外层 SELECT 里找不到为 None。""" + from sqlglot import exp + + from app.engine import cell_format as cf + + try: + select = _outer_select(sql) + if select is None: + return None + item = cf._projection(select, column) + if item is None: + return None + value = cf._unwrap(item.unalias()) + agg: str | None = None + distinct = False + if isinstance(value, exp.AggFunc): + agg = {exp.Sum: "sum", exp.Count: "count", exp.Avg: "avg", exp.Min: "min", exp.Max: "max"}.get( + type(value), "other") + arg = value.this + if isinstance(arg, exp.Distinct): + distinct = True + arg = arg.expressions[0] if len(arg.expressions) == 1 else None + value = cf._unwrap(arg) if arg is not None else None + if not isinstance(value, exp.Column) or isinstance(value.this, exp.Star): + return None, None, agg, distinct + tables = cf._tables(select) + table = tables.get(name_key(value.table)) if value.table else \ + (next(iter(tables.values())) if len(tables) == 1 else None) + return table, value.name, agg, distinct + except Exception: # noqa: BLE001 - 认不出就当没标 + return None + + +def date_source(sql: str, column: str) -> tuple[str, str] | None: + """期次结果列取自哪张表的哪个日期列:那一项选取里只引用了一列(order_date、DATE(order_date)、 + strftime('%Y-%m', order_date))时就是它。认不出为 None。""" + from sqlglot import exp + + from app.engine import cell_format as cf + + try: + select = _outer_select(sql) + item = cf._projection(select, column) if select is not None else None + if item is None: + return None + cols = [c for c in item.unalias().find_all(exp.Column) if not isinstance(c.this, exp.Star)] + if len({(name_key(c.table), name_key(c.name)) for c in cols}) != 1: + return None + ref = cols[0] + tables = cf._tables(select) + table = tables.get(name_key(ref.table)) if ref.table else \ + (next(iter(tables.values())) if len(tables) == 1 else None) + return (table, ref.name) if table else None + except Exception: # noqa: BLE001 + return None + + +class Facts: + """目录事实的取法:run 时 catalogs 是 context["catalogs"](数据源名 → {表名: notes 或 CatalogEntry}),复算时 recorded 是 + 结论里记下的 params["catalog"]。查过的都记下(record),同一个键只认一次。""" + + def __init__(self, catalogs: Mapping[str, Any] | None = None, recorded: Mapping[str, Any] | None = None) -> None: + self._catalogs = catalogs or {} + self._recorded = recorded + self._labels: dict[str, str | None] = {} + self._measures: dict[str, tuple[str | None, bool]] = {} + self._dates: dict[str, str | None] = {} + + def _catalog(self, source: str) -> Mapping[str, Any] | None: + found = self._catalogs.get(source) + return found if isinstance(found, Mapping) else None + + def label(self, snap: Any, column: str) -> str | None: + """结果列给人看的叫法(目录里取自的那一列的中文名);没有为 None。""" + if column in self._labels: + return self._labels[column] + if self._recorded is not None: + value = (self._recorded.get("labels") or {}).get(column) + else: + value = None + src = source_of(snap.sql, column) + if src is not None and src[1] is not None: + item = _item(_table_notes(self._catalog(snap.source), src[0]), src[1], "label") + text = item.get("value") if item else None + value = text.strip() if isinstance(text, str) and text.strip() else None + self._labels[column] = value + return value + + def measure(self, snap: Any, column: str) -> tuple[str | None, bool]: + """结果列的度量类型和有没有人确认过:(flow / stock / ratio / identifier / status / attribute 或 None, 确认过)。""" + if column in self._measures: + return self._measures[column] + if self._recorded is not None: + kind = (self._recorded.get("measures") or {}).get(column) + out = (kind, kind is not None and column not in (self._recorded.get("unconfirmed") or [])) + else: + out = self._live_measure(snap, column) + self._measures[column] = out + return out + + def _live_measure(self, snap: Any, column: str) -> tuple[str | None, bool]: + src = source_of(snap.sql, column) + if src is None: + return None, False + table, name, agg, distinct = src + if agg == "count" and not distinct: + return "flow", True + if agg == "avg": + return "ratio", True + if agg not in (None, "sum") or name is None: + return None, False + item = _item(_table_notes(self._catalog(snap.source), table), name, "measure") + kind = item.get("value") if item else None + if not isinstance(kind, str): + return None, False + return kind, item.get("status") in _STRONG + + def business_date(self, source: str, table: str) -> str | None: + """表的业务日期列(目录的 business_date);没写为 None。""" + key = name_key(table) + if key in self._dates: + return self._dates[key] + if self._recorded is not None: + value = (self._recorded.get("dates") or {}).get(key) + else: + notes = _table_notes(self._catalog(source), table) + item = notes.get("business_date") if isinstance(notes, Mapping) else None + raw = item.get("value") if isinstance(item, Mapping) else None + col = raw.get("column") if isinstance(raw, Mapping) else None + value = col if isinstance(col, str) and col.strip() else None + self._dates[key] = value + return value + + def record(self) -> dict[str, Any] | None: + """查过、查得到的事实(写进 params["catalog"]);一项都没有为 None。""" + out: dict[str, Any] = {} + labels = {k: v for k, v in self._labels.items() if v is not None} + measures = {k: v[0] for k, v in self._measures.items() if v[0] is not None} + unconfirmed = sorted(k for k, v in self._measures.items() if v[0] is not None and not v[1]) + dates = {k: v for k, v in self._dates.items() if v is not None} + for key, value in (("labels", labels), ("measures", measures), ("unconfirmed", unconfirmed), ("dates", dates)): + if value: + out[key] = value + return out or None + + +__all__ = ["Facts", "date_source", "source_of"] diff --git a/backend/app/analysis/inputs.py b/backend/app/analysis/inputs.py new file mode 100644 index 00000000..25d82152 --- /dev/null +++ b/backend/app/analysis/inputs.py @@ -0,0 +1,220 @@ +"""算子的输入:按名字取快照、拆单元格的写法、按列名认列、遮罩列、度量格换成精确的数、同一键同一期多行(细案 01 §1、§8.1)。 + +- 名字:agent 写本节点查过的编号(Q3,和交回模型的「【证据 Q3】」同一个),分析卡写来源节点 id;resolve 取不到就是调用错误。 +- 单元格写成「名字.r行.列」(Q1.r0.gmv),行从 0 数;名字里可以有点(取最后一个「.r数字.」切开)。 +- 列名比较不分大小写,按快照的 columns 定位:一字不差的优先;只差大小写的有好几列时是调用错误。 +- 遮罩列(快照的 masked,不分大小写)不能当维度、度量、期次:调用错误。结论里只出现非遮罩列的格。 +- 度量格:数(含 DECIMAL 落成的数字文本)换成 Fraction;空值为 None(各算子按细案 §1 处理);文本、布尔是调用错误。 + 查询时记了类型的列按类型认:text 列里的 "2026" 不当数。 + +调用错误(contract.CallError)的说法一种,写给 agent:说清哪里不对、怎么改。 +""" +from __future__ import annotations + +import math +import re +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from fractions import Fraction +from typing import Any + +from app.analysis.contract import AnalysisInput, CallError, InputSnapshot +from app.analysis.numbers import canon, to_fraction + +#: 单元格的写法:名字.r行.列 +_CELL = re.compile(r"^(?P.+)\.r(?P\d+)\.(?P.+)$") +#: 报错里列出的列最多几个 +_LIST_MAX = 12 + + +def _listed(columns: Sequence[str]) -> str: + shown = "、".join(columns[:_LIST_MAX]) + return shown + (f"等 {len(columns)} 列" if len(columns) > _LIST_MAX else "") + + +def hashable(value: Any) -> Any: + """维度值当键:文本、数、布尔、空值原样;别的(列表、对象)和非数、无穷写成文字(同 engine/compare._hashable)。""" + if isinstance(value, float) and not math.isfinite(value): + return repr(value) + return value if isinstance(value, (str, int, float, bool)) or value is None else repr(value) + + +def same_value(a: Any, b: Any) -> bool: + """调用里写的期次、段和快照里的值是不是同一个:相等,或者写出来一样("2026" 和 2026、" 华东" 和 "华东")。""" + if a == b and type(a) is type(b): + return True + if isinstance(a, bool) or isinstance(b, bool): + return a == b + fa, fb = to_fraction(a), to_fraction(b) + if fa is not None and fb is not None and not isinstance(a, str) and not isinstance(b, str): + return fa == fb + return str(a).strip() == str(b).strip() + + +@dataclass(eq=False) +class Snap: + """一份输入快照(InputSnapshot 的薄包装)。""" + + ref: str + data: Mapping[str, Any] + dialect: str | None = None + columns: list[str] = field(default_factory=list) + rows: list[list[Any]] = field(default_factory=list) + + def __post_init__(self) -> None: + self.columns = [str(c) for c in self.data.get("columns") or []] + self.rows = [list(r) if isinstance(r, (list, tuple)) else [] for r in self.data.get("rows") or []] + + @property + def artifact(self) -> str | None: + value = self.data.get("artifact") + return value if isinstance(value, str) and value else None + + @property + def source(self) -> str: + return str(self.data.get("source") or "") + + @property + def sql(self) -> str: + return str(self.data.get("sql") or "") + + @property + def truncated(self) -> bool: + return bool(self.data.get("truncated")) + + @property + def masked(self) -> set[str]: + """不能当维度、度量的列(小写):入口交来的 masked;没交(复算时取回的是快照工件本身)就按快照记下的遮罩列 + (数据源设的 mask_columns、查询时整列遮掉的 masked_at_query)。""" + found = self.data.get("masked") + if found is None: + found = [*(self.data.get("mask_columns") or []), *(self.data.get("masked_at_query") or [])] + return {str(m).strip().lower() for m in found or [] if str(m).strip()} + + def kind(self, column: str) -> str | None: + """查询时记下的这一列的类型(column_types 是 {列: 类型} 或按列序的列表)。""" + types = self.data.get("column_types") + if isinstance(types, Mapping): + kind = types.get(column) + elif isinstance(types, list) and column in self.columns: + i = self.columns.index(column) + kind = types[i] if i < len(types) else None + else: + kind = None + return kind if isinstance(kind, str) else None + + def column(self, wanted: Any, role: str) -> str: + """按名字认一列(不分大小写),认出的是快照里的写法。不存在、只差大小写的好几列、遮罩列都是调用错误。""" + if not isinstance(wanted, str) or not wanted.strip(): + raise CallError("missing_column", f"没写{role}:写 {self.ref} 里的列名") + name = wanted.strip() + hits = [c for c in self.columns if c == name] or [c for c in self.columns if c.lower() == name.lower()] + if not hits: + raise CallError("unknown_column", f"{self.ref} 里没有「{name}」这一列,{role}要写它的列名:{self.ref} 的列有 " + f"{_listed(self.columns)}") + if len(hits) > 1: + raise CallError("ambiguous_column", f"{self.ref} 里有好几列都叫「{name}」(只差大小写):{role}写出确切的列名") + column = hits[0] + if column.lower() in self.masked: + raise CallError("masked_column", f"「{column}」是遮罩列,不能当{role}:换一列,或者在 SQL 里不选它") + return column + + def index(self, column: str) -> int: + return self.columns.index(column) + + def cell(self, row: int, column: str) -> Any: + r = self.rows[row] + i = self.index(column) + return r[i] if i < len(r) else None + + def number(self, row: int, column: str, role: str) -> Fraction | None: + """一格换成精确的数;空值为 None。不是数是调用错误。""" + raw = self.cell(row, column) + if raw is None: + return None + if isinstance(raw, bool) or (isinstance(raw, str) and self.kind(column) == "text"): + raise self._not_numeric(row, column, raw, role) + if isinstance(raw, str): + from app.engine.expressions import numeric_text + + if numeric_text(raw.strip()) is None: + raise self._not_numeric(row, column, raw, role) + value = to_fraction(raw) + if value is None: + raise self._not_numeric(row, column, raw, role) + return value + + def _not_numeric(self, row: int, column: str, raw: Any, role: str) -> CallError: + shown = str(raw).replace("\n", " ")[:20] + return CallError("not_numeric", f"{self.ref} 的「{column}」不是数值列(第 {row} 行是「{shown}」):{role}要选数值列") + + +class Inputs: + """按名字取输入,记下结论的 inputs(同一份快照、同一格只记一次,按用到的先后)。""" + + def __init__(self, resolve: Callable[[str], InputSnapshot | None], + dialect: Callable[[str, Mapping[str, Any]], str | None] | None = None) -> None: + self._resolve = resolve + self._dialect = dialect + self._snaps: dict[str, Snap] = {} + self.records: list[AnalysisInput] = [] + + def snap(self, name: Any, role: str) -> Snap: + """按名字取一份快照。""" + if not isinstance(name, str) or not name.strip(): + raise CallError("missing_query", f"没写{role}:写本节点查过的查询编号(如 Q3)") + key = name.strip() + if key not in self._snaps: + found = self._resolve(key) + if not isinstance(found, Mapping): + raise CallError("unknown_query", f"没有叫「{key}」的查询结果:{role}写本节点查过的查询编号(如 Q3)") + self._snaps[key] = Snap(key, found, self._dialect(key, found) if self._dialect else found.get("dialect")) + return self._snaps[key] + + def cell(self, ref: Any, role: str, label: str) -> tuple[Snap, int, str, Fraction | None]: + """「名字.r行.列」→ (快照, 行, 列, 值)。""" + found = _CELL.match(ref.strip()) if isinstance(ref, str) else None + if found is None: + raise CallError("bad_cell", f"{label}要写成单元格「编号.r行.列」(如 Q1.r0.gmv),现在是「{ref}」") + snap = self.snap(found.group("name"), label) + row = int(found.group("row")) + if row >= len(snap.rows): + raise CallError("row_out_of_range", f"{snap.ref} 没有第 {row} 行(共 {len(snap.rows)} 行,从 r0 数起):改{label}") + column = snap.column(found.group("column"), label) + value = snap.number(row, column, label) + self.record({"ref": f"{snap.ref}.r{row}.{column}", "role": role, "row": row, "column": column, + "value": canon(value)}, snap) + return snap, row, column, value + + def use(self, snap: Snap, role: str, columns: Sequence[str], rows: Sequence[int] | None = None) -> None: + """记下一份快照的用法:用了哪几列、哪几行(全用了写 all)。""" + used = sorted(set(rows)) if rows is not None else None + shown: Any = "all" if used is None or used == list(range(len(snap.rows))) else used + self.record({"ref": snap.ref, "role": role, "columns": list(dict.fromkeys(columns)), "rows": shown}, snap) + + def record(self, entry: dict[str, Any], snap: Snap) -> None: + out: dict[str, Any] = {"ref": entry["ref"]} + if snap.artifact: + out["artifact"] = snap.artifact + if snap.dialect: + out["dialect"] = snap.dialect + out.update({k: v for k, v in entry.items() if k != "ref"}) + if out not in self.records: + self.records.append(out) # type: ignore[arg-type] + + def parent(self, name: str, artifact: str) -> None: + entry: dict[str, Any] = {"ref": name, "artifact": artifact, "role": "parent"} + if entry not in self.records: + self.records.append(entry) # type: ignore[arg-type] + + +def as_list(value: Any) -> list[Any]: + """参数里一个值或一组值:都规范成列表(None 是空列表)。""" + if value is None: + return [] + if isinstance(value, (list, tuple)): + return list(value) + return [value] + + +__all__ = ["Inputs", "Snap", "as_list", "hashable", "same_value"] diff --git a/backend/app/analysis/periods.py b/backend/app/analysis/periods.py new file mode 100644 index 00000000..2f44d710 --- /dev/null +++ b/backend/app/analysis/periods.py @@ -0,0 +1,224 @@ +"""期次:快照里期次列的值认成可以排序、能判断连不连续、知道覆盖哪几天的期(细案 01 §3「期次认得出、能排序」)。 + +认这几种写法(一列里要是同一种): + +- 日期:DATE / DATETIME(零点)、'YYYY-MM-DD'、'YYYYMMDD'、8 位整数。粒度不写时按值推断:都是某月 1 日(或都是月末) + 且相邻两期隔整月 → month(月份都在 1、4、7、10 月且隔整季 → quarter);相邻两期都隔 7 天的整数倍 → week(这个日期当 + 一周的第一天);否则 day。 +- 月份:'YYYY-MM'、'YYYYMM'(6 位,月份 01–12)→ month。 +- ISO 周:'YYYY-Www'(2026-W42 是 2026-10-12 到 10-18)→ week。SQLite 的 strftime('%W') 不是 ISO 周号(年初那几天 + 算第 0 周),按它写的期次覆盖的日子会差一周:方法里要求期次写成每周第一天的日期。 +- 季度:'YYYY-Qn'、'YYYYQn' → quarter。 +- 整数序号(不是上面几种的整数)→ seq:能排序、能判连续,不知道覆盖哪几天(本期完不完整核对不了)。 + +带时刻(非零点)的、写法混着的、认不出的都算认不出(period_unreadable)。 + +每一期记 ordinal:同一粒度里相邻两期的 ordinal 差 1(日:天数,周:从第一期起第几个 7 天,月:年 × 12 + 月,季:年 × 4 + +季,序号:本身),据此排序、判缺期、按 lag 找比较期。start / end 是这一期覆盖的第一天和最后一天(含),seq 没有。 +""" +from __future__ import annotations + +import calendar +import re +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from typing import Any + +from app.data import coverage + +GRAINS = ("day", "week", "month", "quarter") +#: 粒度的叫法:数量词(过去 12 周、12 个月)、每…(每周、每月) +COUNT_WORD = {"day": "天", "week": "周", "month": "个月", "quarter": "个季度", "seq": "期"} +PER_WORD = {"day": "天", "week": "周", "month": "月", "quarter": "季度", "seq": "期"} +#: 粒度不写时各粒度的缺省 lag:日数据和上周同一天比(避开星期效应),其余和上一期比 +DEFAULT_LAG = {"day": 7, "week": 1, "month": 1, "quarter": 1, "seq": 1} + +_MONTH = re.compile(r"^(\d{4})-?(\d{2})$") +_WEEK = re.compile(r"^(\d{4})-?W(\d{1,2})$", re.I) +_QUARTER = re.compile(r"^(\d{4})-?Q([1-4])$", re.I) + + +@dataclass(frozen=True) +class Period: + raw: Any # 快照里的原值 + ordinal: int + start: date | None + end: date | None + + +def _month_end(d: date) -> date: + return d.replace(day=calendar.monthrange(d.year, d.month)[1]) + + +def _quarter_start(d: date) -> date: + return date(d.year, 3 * ((d.month - 1) // 3) + 1, 1) + + +def _quarter_end(d: date) -> date: + start = _quarter_start(d) + return _month_end(date(start.year, start.month + 2, 1)) + + +def _as_date(value: Any) -> date | None: + """日期写法的期次 → 那一天;带非零点时刻的、认不出的为 None。""" + if isinstance(value, datetime): + return value.date() if value.time() == datetime.min.time() else None + if isinstance(value, date): + return value + if isinstance(value, bool): + return None + day = coverage.as_day(value) + if day is None or day.timed: + return None + return day.day + + +def _shape(value: Any) -> str | None: + """一个期次值是哪种写法:date / month / week / quarter / seq;认不出为 None。""" + if isinstance(value, bool) or value is None: + return None + if isinstance(value, (date, datetime)): + return "date" if _as_date(value) else None + if isinstance(value, float) and value.is_integer(): + value = int(value) + if isinstance(value, int): + text = str(value) + if len(text) == 8 and _as_date(value): + return "date" + if len(text) == 6 and (m := _MONTH.match(text)) and 1 <= int(m.group(2)) <= 12: + return "month" + return "seq" + if not isinstance(value, str): + return None + text = value.strip() + if (m := _MONTH.match(text)) and 1 <= int(m.group(2)) <= 12 and 1900 <= int(m.group(1)) <= 2200: + return "month" + if _WEEK.match(text): + return "week" + if _QUARTER.match(text): + return "quarter" + if _as_date(text): + return "date" + if re.fullmatch(r"-?\d+", text): + return "seq" + return None + + +def _month_of(value: Any) -> date: + m = _MONTH.match(str(int(value) if isinstance(value, (int, float)) else value).strip()) + assert m is not None + return date(int(m.group(1)), int(m.group(2)), 1) + + +def _infer_date_grain(days: Sequence[date]) -> str: + """一列日期的粒度(见文件头)。""" + ordered = sorted(set(days)) + if len(ordered) < 2: + return "day" + firsts = all(d.day == 1 for d in ordered) + lasts = all(d == _month_end(d) for d in ordered) + if firsts or lasts: + months = [d.year * 12 + d.month - 1 for d in ordered] + gaps = [b - a for a, b in zip(months, months[1:])] + if all(g >= 1 for g in gaps): + if all(d.month in (1, 4, 7, 10) for d in ordered) and all(g % 3 == 0 for g in gaps) and firsts: + return "quarter" + return "month" + gaps = [(b - a).days for a, b in zip(ordered, ordered[1:])] + if all(g % 7 == 0 for g in gaps): + return "week" + return "day" + + +def parse(values: Sequence[Any], grain: str | None = None) -> tuple[list[Period], str] | None: + """一列期次值 → (逐个的 Period,粒度)。认不出(写法不一、带时刻、粒度和写法对不上)返回 None。""" + shapes = {_shape(v) for v in values} + if not values or None in shapes or len(shapes) != 1: + return None + shape = shapes.pop() + if shape == "seq": + if grain not in (None, "seq"): + return None + return [Period(v, int(v), None, None) for v in values], "seq" + if shape == "month": + if grain not in (None, "month"): + return None + out = [] + for v in values: + start = _month_of(v) + out.append(Period(v, start.year * 12 + start.month - 1, start, _month_end(start))) + return out, "month" + if shape == "week": + if grain not in (None, "week"): + return None + out = [] + for v in values: + m = _WEEK.match(str(v).strip()) + assert m is not None + try: + start = date.fromisocalendar(int(m.group(1)), int(m.group(2)), 1) + except ValueError: + return None + out.append(Period(v, start.toordinal() // 7, start, start + timedelta(days=6))) + return out, "week" + if shape == "quarter": + if grain not in (None, "quarter"): + return None + out = [] + for v in values: + m = _QUARTER.match(str(v).strip()) + assert m is not None + start = date(int(m.group(1)), 3 * (int(m.group(2)) - 1) + 1, 1) + out.append(Period(v, start.year * 4 + (start.month - 1) // 3, start, _quarter_end(start))) + return out, "quarter" + days = [_as_date(v) for v in values] + assert all(days) + grain = grain or _infer_date_grain(days) # type: ignore[arg-type] + if grain == "day": + return [Period(v, d.toordinal(), d, d) for v, d in zip(values, days)], "day" # type: ignore[union-attr] + if grain == "week": + first = min(days) # type: ignore[type-var] + if any((d - first).days % 7 for d in days): # type: ignore[operator] + return None + return [Period(v, (d - first).days // 7, d, d + timedelta(days=6)) # type: ignore[operator] + for v, d in zip(values, days)], "week" + if grain == "month": + if not all(d.day == 1 or d == _month_end(d) for d in days): # type: ignore[union-attr] + return None + return [Period(v, d.year * 12 + d.month - 1, d.replace(day=1), _month_end(d)) # type: ignore[union-attr] + for v, d in zip(values, days)], "month" + if grain == "quarter": + if not all(d.day == 1 and d.month in (1, 4, 7, 10) for d in days): # type: ignore[union-attr] + return None + return [Period(v, d.year * 4 + (d.month - 1) // 3, d, _quarter_end(d)) # type: ignore[union-attr] + for v, d in zip(values, days)], "quarter" + return None + + +def calendar_unit(start: date, end: date) -> str | None: + """这段日子是不是一个整的日历期:整月 month、整季 quarter、整年 year;都不是为 None。整月对整月算一样长(31 天的 10 月 + 和 30 天的 9 月环比是通行的比法),不按天数判窗口不等。""" + if start.day != 1: + return None + if start.month == 1 and end == date(start.year, 12, 31): + return "year" + if start.month in (1, 4, 7, 10) and end == _quarter_end(start): + return "quarter" + if end == _month_end(start): + return "month" + return None + + +def weekend_days(start: date, end: date) -> int: + """[start, end] 里周六、周日的天数。""" + days = (end - start).days + 1 + full, rest = divmod(days, 7) + count = 2 * full + for k in range(rest): + if (start + timedelta(days=7 * full + k)).weekday() >= 5: + count += 1 + return count + + +__all__ = ["COUNT_WORD", "DEFAULT_LAG", "GRAINS", "PER_WORD", "Period", "calendar_unit", "parse", "weekend_days"] diff --git a/backend/app/analysis/probe.py b/backend/app/analysis/probe.py new file mode 100644 index 00000000..ade122d1 --- /dev/null +++ b/backend/app/analysis/probe.py @@ -0,0 +1,254 @@ +"""可比性检查的现查(细案 01 §1「只读快照、不发查询」的唯一例外,§6):表里这一列的实际日期范围、按运行当天现算的界。 + +算子本身不查库。要现查的只有两样,都走 data/coverage 现成的只读探查(只读守卫、遮罩列不探、每条 PROBE_TIMEOUT_S 秒、总 +时限、同一张表 5 分钟内不重查、现查不成退回剖析事实): + +- 范围:(数据源名, 表, 列) → contract.Probe {source, table, column, min, max, basis, at},min / max 是 ISO 日期;表里这一列 + 一个值都没有时 min、max 都为 None(basis live)。basis 是 live(现查)/ profile(按剖析)/ observed(按取值观察),at 是现查的 + 时刻(服务端时区,ISO,秒)或剖析、观察的日期。 +- 界:SQL 里按运行当天现算的界(CURRENT_DATE、DATE_SUB(…)、date('now', …))在同一个库里算出来:(数据源名, 界的 SQL 写法) + → Probe {source, expr, value, basis: live, at},value 是库里算出的日期(带时刻的写成「YYYY-MM-DD HH:MM:SS」)。 + +**结论里记下用到的现查结果,复算用记下的值**:Book 是一次分析的现查簿。run 时算子第一遍只登记要查什么(拿到的都是「不 +知道」),入口查完再算一遍,直到不再要新的(算子在拿到范围之后才可能要界,最多三轮);复算时 Book 里装的是结论记下的 +probes,不现查。结论的 probes 只记最后一遍真用到、查得到的,按用到的先后——同样的输入、同样的现查结果,结论逐字节相同。 + +入口(agent 分析工具、分析卡、口径卡的可比性挂钩)用 make_prober 生成 contract.Prober 交进 context["probe"],见它的说明。 +""" +from __future__ import annotations + +import time +from collections.abc import Awaitable, Callable, Iterable, Mapping +from datetime import datetime +from typing import Any + +from app.analysis.contract import Probe +from app.data.names import name_key + +#: 算子在拿到现查结果之后可能还要新的(先要范围、再要界):run 最多这么多轮 +MAX_ROUNDS = 3 + + +def _range_key(source: str, table: str, column: str) -> tuple[str, str, str, str]: + return ("range", source or "", name_key(str(table)), name_key(str(column))) + + +def _bound_key(source: str, expr: str) -> tuple[str, str, str]: + return ("bound", source or "", str(expr)) + + +def probe_key(probe: Mapping[str, Any]) -> tuple[str, ...]: + """一条记下的现查结果的键(Book 按它查)。""" + if probe.get("expr") is not None: + return _bound_key(str(probe.get("source") or ""), str(probe["expr"])) + return _range_key(str(probe.get("source") or ""), str(probe.get("table") or ""), str(probe.get("column") or "")) + + +class Book: + """一次分析的现查簿。known:{键: Probe 或 None(查过、查不到)}。算子经 range / bound 取;asked 是按先后登记的键,missing + 是登记了、簿里还没有的(run 据此去查),used 是最后真用到、查得到的(写进结论的 probes)。""" + + def __init__(self, known: Mapping[tuple[str, ...], Probe | None] | Iterable[Probe] = ()) -> None: + if isinstance(known, Mapping): + self._known: dict[tuple[str, ...], Probe | None] = dict(known) + self._closed = False + else: + self._known = {probe_key(p): p for p in known} + self._closed = True # 复算:记下的就是全部,没记的当查不到 + self.asked: list[tuple[str, ...]] = [] + + def _ask(self, key: tuple[str, ...]) -> Probe | None: + if key not in self.asked: + self.asked.append(key) + return self._known.get(key) + + def range(self, source: str, table: str, column: str) -> Probe | None: + return self._ask(_range_key(source, table, column)) + + def bound(self, source: str, expr: str) -> Probe | None: + return self._ask(_bound_key(source, expr)) + + @property + def missing(self) -> list[tuple[str, ...]]: + return [] if self._closed else [k for k in self.asked if k not in self._known] + + @property + def used(self) -> list[Probe]: + return [p for p in (self._known.get(k) for k in self.asked) if p is not None] + + +def _now() -> str: + return datetime.now().astimezone().replace(tzinfo=None).isoformat(timespec="seconds") + + +async def fetch(book: Book, context: Mapping[str, Any]) -> dict[tuple[str, ...], Probe | None]: + """把簿里登记了、还没有的现查一遍:先 context["probe"],查不到的范围退回 context["facts"](数据源名 → {表名: 剖析事实}, + 或数据源名 → 取剖析事实的函数),都没有记 None。交回新查到的那几条。""" + from app.data import coverage + + prober = context.get("probe") + facts = context.get("facts") or {} + out: dict[tuple[str, ...], Probe | None] = {} + for key in book.missing: + found: Probe | None = None + if key[0] == "range": + _, source, table, column = key + if prober is not None: + try: + found = await prober(source, table, column) + except Exception: # noqa: BLE001 - 探查失败就当不知道 + found = None + if found is None: + found = await _from_facts(facts.get(source), source, table, column, coverage) + else: + _, source, expr = key + bounds = getattr(prober, "bounds", None) + if bounds is not None: + try: + found = await bounds(source, expr) + except Exception: # noqa: BLE001 + found = None + out[key] = found + return out + + +async def _from_facts(facts: Any, source: str, table: str, column: str, coverage: Any) -> Probe | None: + """探查不成时按剖析事实兜底:facts 是 {表名: last_facts 的那一项} 或取它的函数(coverage.Facts)。""" + found: Any = None + try: + if callable(facts): + found = await facts(table) + elif isinstance(facts, Mapping): + found = next((v for k, v in facts.items() if name_key(str(k)) == name_key(table) + or name_key(str(k).rsplit(".", 1)[-1]) == name_key(table.rsplit(".", 1)[-1])), None) + except Exception: # noqa: BLE001 + return None + data = coverage.facts_range(found, column) + return _range_probe(source, table, column, data) if data else None + + +def _range_probe(source: str, table: str, column: str, data: Mapping[str, Any]) -> Probe: + basis = data.get("basis") if data.get("basis") in ("live", "profile", "observed") else "live" + at = _now() if basis == "live" else str(data.get("at") or "") + return {"source": source, "table": table, "column": column, "min": data.get("min"), "max": data.get("max"), + "basis": basis, "at": at} # type: ignore[typeddict-item] + + +class LiveProber: + """contract.Prober:await prober(数据源名, 表, 列) → Probe | None;另有 await prober.bounds(数据源名, 界的 SQL) 在库里算界, + prober.dialect(数据源名) 是这个源的 sqlglot 方言(算子按它读 SQL、写界)。见 make_prober。""" + + def __init__(self, sources: Mapping[str, Any], *, masked: Mapping[str, Iterable[str]] | None, + facts: Mapping[str, Callable[[str], Awaitable[Mapping[str, Any] | None]]] | None, + cache: Mapping[str, Any] | None, budget: float, + clock: Callable[[], str] | None) -> None: + from app.data import coverage + + self._sources = dict(sources) + self._masked = {k: {str(c).strip().lower() for c in v} for k, v in (masked or {}).items()} + self._facts = dict(facts or {}) + self._cache: dict[str, Any] = dict(cache or {}) + self._budget = budget + self._deadline: float | None = None + self._clock = clock or _now + self._bounds: dict[tuple[str, str], Any] = {} + self._coverage = coverage + + def dialect(self, source: str) -> str | None: + from app.data.sqlcheck import dialect_of + + found = self._sources.get(source) + return dialect_of(getattr(found, "kind", None)) if found is not None else None + + def _expired(self) -> bool: + if self._deadline is None: + self._deadline = time.monotonic() + self._budget + return time.monotonic() > self._deadline + + def _hidden(self, source: Any, name: str) -> set[str]: + if name in self._masked: + return self._masked[name] + from app.data.engine import masked_columns + + return {c.lower() for c in masked_columns(getattr(source, "options", None) or {})} + + def _facts_of(self, name: str, source: Any) -> Callable[[str], Awaitable[Mapping[str, Any] | None]]: + return self._facts.get(name) or profile_facts(source) + + async def __call__(self, source: str, table: str, column: str) -> Probe | None: + found = self._sources.get(source) + if found is None or str(column).strip().lower() in self._hidden(found, source): + return None + if self._expired(): + return None + cache = self._cache.setdefault(source, self._coverage.Cache()) + data = await self._coverage.column_range(found, table, column, facts=self._facts_of(source, found), cache=cache) + if data is None: + return None + probe = _range_probe(source, table, column, data) + if probe["basis"] == "live": + probe["at"] = self._clock() + return probe + + async def bounds(self, source: str, expr: str) -> Probe | None: + found = self._sources.get(source) + if found is None or self._expired(): + return None + key = (source, expr) + if key not in self._bounds: + values = await self._coverage.bound_values(found, [expr]) + self._bounds[key] = values[0] if values else None + value = self._bounds[key] + if value is None: + return None + return {"source": source, "expr": expr, "value": str(value), "basis": "live", # type: ignore[typeddict-unknown-key] + "at": self._clock()} + + +def profile_facts(source: Any) -> Callable[[str], Awaitable[Mapping[str, Any] | None]]: + """这个数据源上一张表的剖析事实(catalog_profile_jobs.last_facts 的那一项,同 tools/datasource._coverage 的退回):上传 + 表格的快照对不上这次读的那一版时不用。""" + async def facts(table: str) -> Mapping[str, Any] | None: + from app.data import catalog as data_catalog + from app.data.catalog_profile_jobs import last_facts + from app.db.base import SessionLocal + + source_id = getattr(source, "id", None) + if not source_id: + return None + name = data_catalog.resolve_table_name(getattr(source, "schema_cache", None), table) or table + async with SessionLocal() as session: + found = (await last_facts(session, str(source_id), [name])).get(name) + snapshot = getattr(source, "snapshot_id", None) + if found and snapshot and found.get("snapshot_id") != snapshot: + return None + return found + + return facts + + +def make_prober(sources: Mapping[str, Any], *, masked: Mapping[str, Iterable[str]] | None = None, + facts: Mapping[str, Callable[[str], Awaitable[Mapping[str, Any] | None]]] | None = None, + cache: Mapping[str, Any] | None = None, budget: float | None = None, + clock: Callable[[], str] | None = None) -> LiveProber: + """生成一次分析调用用的探查器(contract.Prober,交进 context["probe"])。 + + - sources:数据源名 → 数据源对象(DataSource 行,或上传源绑定快照的视图:同 tools/datasource 建工具时的 source,要有 + name、kind、options、schema_cache;上传源另有 snapshot_id)。名字和快照里的 source 一致。没给的源不探。 + - masked:数据源名 → 遮罩列(不分大小写)。缺省按数据源 options 里设的遮罩列(data/engine.masked_columns)。遮罩列上 + 不探,探查器直接交回 None。 + - facts:数据源名 → 取剖析事实的函数(coverage.Facts:表名 → last_facts 的那一项)。缺省按 profile_facts 现读库(同查询 + 工具的覆盖检查)。 + - cache:数据源名 → coverage.Cache(同一张表、同一个界一次运行里只查一次);不给就每个探查器各自一份。另外 coverage + 在进程里把现查到的范围记 5 分钟。 + - budget:这个探查器所有探查加起来最多多少秒(缺省 coverage.BUDGET_S,从第一次探查算起);过了的一律交回 None。一次 + 分析调用建一个。 + - clock:现查时刻的写法(测试用);缺省是服务端时区的 ISO 时间(到秒)。 + """ + from app.data import coverage + + return LiveProber(sources, masked=masked, facts=facts, cache=cache, + budget=coverage.BUDGET_S if budget is None else budget, clock=clock) + + +__all__ = ["Book", "LiveProber", "MAX_ROUNDS", "fetch", "make_prober", "probe_key", "profile_facts"] diff --git a/backend/app/data/coverage.py b/backend/app/data/coverage.py index e7db5445..e30761b9 100644 --- a/backend/app/data/coverage.py +++ b/backend/app/data/coverage.py @@ -447,9 +447,14 @@ async def _fallback(flt: _Filter, facts: Facts) -> _Range | None: found = await facts(flt.table_name) except Exception: # noqa: BLE001 return None + return _facts_range(found, flt.column_name) + + +def _facts_range(found: Any, column: str) -> _Range | None: + """剖析事实(last_facts 的一项)里这一列的范围:先剖析的 date_ranges,再取值观察的最新日期。""" if not isinstance(found, Mapping): return None - wanted = name_key(flt.column_name) + wanted = name_key(column) for item in found.get("date_ranges") or []: if isinstance(item, Mapping) and name_key(str(item.get("column") or "")) == wanted: lo, hi = _as_day(item.get("min")), _as_day(item.get("max")) @@ -806,5 +811,88 @@ def no_data_of(snapshot: Any, column: Any) -> dict[str, Any] | None: return None -__all__ = ["BUDGET_S", "Cache", "NO_DATA_SHORT", "PROBE_TIMEOUT_S", "Window", "filters", "narrowed", "no_data_of", - "result_gaps", "sql_gaps", "zero_columns"] +# ========================================================================== +# 给分析的可比性检查用的几个小函数(app/analysis/comparable.py、probe.py:窗口长度、整期)。读法、写法、探查都和上面同一套, +# 只是把中间结果交出去:分析结论要把现查的范围、在库里算出的界记下来,复算时用记下的值 +# ========================================================================== + + +def parse_sql(sql: Any, read: str | None) -> Any: + """一条查询解析成语法树(同覆盖检查:太长、带模板占位、不止一条、不是查询的为 None)。""" + return _parse(sql, read) + + +def as_day(value: Any, kind: str | None = None) -> _Day | None: + """一个值归一成日期(DATE / DATETIME、'YYYY-MM-DD'、'YYYYMMDD'、8 位整数);认不出为 None。""" + return _as_day(value, kind) + + +def literal_bound(node: Any) -> tuple[bool, _Day | None]: + """时间条件的一个界:(是不是字面量, 字面量归一成的日期)。不是字面量的是按运行当天现算的,要在库里算。""" + return (True, _literal_day(node)) if _is_literal(node) else (False, None) + + +def window_of(conds: list[tuple[str, Any, bool]], days: Mapping[int, _Day | None]) -> Window | None: + """几个条件合成一段区间(同 _window:下界取最晚、上界取最早,「>」「<」的界不带时刻时按天挪一天)。days 是 + {id(界的节点): 日期}。不核对列和界的写法比不比得起来:分析结论里表的范围只记了日期。""" + return _window(conds, days, _Range(None, None, "none"), None) + + +def _range_out(data: _Range | None) -> dict[str, Any] | None: + if data is None: + return None + return {"min": data.low.day.isoformat() if data.low else None, + "max": data.high.day.isoformat() if data.high else None, + "basis": data.basis, "at": data.day, "empty": data.empty} + + +def facts_range(found: Any, column: str) -> dict[str, Any] | None: + """剖析事实(catalog_profile_jobs.last_facts 的一项)里这一列的范围,形状同 column_range;没有为 None。""" + return _range_out(_facts_range(found, column)) + + +async def column_range(source: Any, table: str, column: str, *, facts: Facts | None = None, + cache: Cache | None = None) -> dict[str, Any] | None: + """这一列在这张表里的实际范围:现查 MIN / MAX(只读守卫、PROBE_TIMEOUT_S 秒、同一张表 LIVE_TTL_S 秒内不重查),查不成 + 退回剖析、取值观察(facts)。{min, max, basis, at, empty}:min / max 是 ISO 日期(不知道的那一头为 None),basis 是 live / + profile / observed,at 是剖析、观察的日期(现查为 None),表里这一列一个值都没有时 empty 为真。都没有为 None。从不抛异常。""" + from sqlglot import exp + + try: + parts = [p for p in str(table).split(".") if p] + node = exp.Table(this=exp.to_identifier(parts[-1]), + **({"db": exp.to_identifier(parts[-2])} if len(parts) > 1 else {})) + flt = _Filter((), node, str(table), exp.column(str(column)), str(column)) + return _range_out(await _range(source, flt, _read(getattr(source, "kind", None)), cache or Cache(), facts)) + except Exception: # noqa: BLE001 - 探查失败就当不知道 + logger.info("可比性检查的范围探查没有成功(数据源 %s)", getattr(source, "name", ""), exc_info=True) + return None + + +async def bound_values(source: Any, texts: list[str]) -> list[Any] | None: + """按运行当天现算的几个界(SQL 表达式,按这个数据源的方言写)在库里算出来:一条 SELECT 界1, 界2 …(Oracle 加 FROM + DUAL),走只读守卫和 PROBE_TIMEOUT_S。交回各个值(日期、时刻按 ISO 写,其余照库里交回的);查不成为 None。从不抛异常。""" + if not texts: + return [] + read = _read(getattr(source, "kind", None)) + try: + found = await _probe(source, _select_sql([f"{t} AS b{i}" for i, t in enumerate(texts)], read)) + except Exception: # noqa: BLE001 + return None + if found is None: + return None + values = list(found[0]) + [None] * len(texts) + out: list[Any] = [] + for value in values[:len(texts)]: + if isinstance(value, datetime): + out.append(value.isoformat(sep=" ", timespec="seconds")) + elif isinstance(value, date): + out.append(value.isoformat()) + else: + out.append(value) + return out + + +__all__ = ["BUDGET_S", "Cache", "NO_DATA_SHORT", "PROBE_TIMEOUT_S", "Window", "as_day", "bound_values", "column_range", + "facts_range", "filters", "literal_bound", "narrowed", "no_data_of", "parse_sql", "result_gaps", "sql_gaps", + "window_of", "zero_columns"] diff --git a/backend/tests/test_analysis_periods.py b/backend/tests/test_analysis_periods.py new file mode 100644 index 00000000..eee79dde --- /dev/null +++ b/backend/tests/test_analysis_periods.py @@ -0,0 +1,57 @@ +"""期次(app/analysis/periods.py):日期、YYYY-MM、YYYY-Www、YYYY-Qn、整数序号认得出、能排序、知道覆盖哪几天;粒度按值推断; +写法混着、带时刻的认不出。""" +from __future__ import annotations + +from datetime import date + +from app.analysis import periods as pd + + +def test_dates_infer_their_grain(): + items, grain = pd.parse(["2026-10-01", "2026-10-02", "2026-10-04"]) + assert grain == "day" and [i.start for i in items] == [date(2026, 10, 1), date(2026, 10, 2), date(2026, 10, 4)] + items, grain = pd.parse(["2026-09-28", "2026-10-05", "2026-10-19"]) # 隔 7 天的整数倍:周(缺一周) + assert grain == "week" and [i.ordinal for i in items] == [0, 1, 3] and items[0].end == date(2026, 10, 4) + items, grain = pd.parse(["2026-08-01", "2026-09-01", "2026-10-01"]) + assert grain == "month" and items[1].end == date(2026, 9, 30) + items, grain = pd.parse(["2026-01-01", "2026-04-01", "2026-07-01"]) + assert grain == "quarter" and items[0].end == date(2026, 3, 31) + items, grain = pd.parse([20261001, 20261002]) + assert grain == "day" and items[1].start == date(2026, 10, 2) + assert pd.parse(["2026-10-01 00:00:00", "2026-10-02 00:00:00"])[1] == "day" + + +def test_labels_for_months_weeks_quarters_and_sequences(): + items, grain = pd.parse(["2026-09", "2026-10"]) + assert grain == "month" and items[1].ordinal - items[0].ordinal == 1 and items[1].end == date(2026, 10, 31) + items, grain = pd.parse([202609, 202610]) + assert grain == "month" + items, grain = pd.parse(["2026-W41", "2026-W42", "2026-W01"]) + assert grain == "week" and items[1].start == date(2026, 10, 12) and items[1].end == date(2026, 10, 18) + assert items[1].ordinal - items[0].ordinal == 1 + items, grain = pd.parse(["2025-W52", "2026-W01"]) # 跨年也连着 + assert items[1].ordinal - items[0].ordinal == 1 + items, grain = pd.parse(["2026Q3", "2026-Q4"]) + assert grain == "quarter" and items[1].start == date(2026, 10, 1) + items, grain = pd.parse([3, 1, 2]) + assert grain == "seq" and items[0].start is None and [i.ordinal for i in items] == [3, 1, 2] + + +def test_unreadable_periods(): + assert pd.parse(["2026-10-01", "2026-W41"]) is None # 写法混着 + assert pd.parse(["2026-10-01 08:30:00", "2026-10-02 08:30:00"]) is None # 带时刻 + assert pd.parse(["上周", "本周"]) is None + assert pd.parse(["2026-10", "2026-11"], "week") is None # 粒度和写法对不上 + assert pd.parse(["2026-10-01", "2026-10-03"], "week") is None # 按周但不隔 7 天 + assert pd.parse([]) is None and pd.parse([None, "2026-10-01"]) is None + + +def test_calendar_units_and_weekend_days(): + assert pd.calendar_unit(date(2026, 9, 1), date(2026, 9, 30)) == "month" + assert pd.calendar_unit(date(2026, 7, 1), date(2026, 9, 30)) == "quarter" + assert pd.calendar_unit(date(2026, 1, 1), date(2026, 12, 31)) == "year" + assert pd.calendar_unit(date(2026, 9, 2), date(2026, 9, 30)) is None + assert pd.weekend_days(date(2026, 10, 12), date(2026, 10, 18)) == 2 # 周一到周日 + assert pd.weekend_days(date(2026, 10, 16), date(2026, 10, 18)) == 2 # 周五到周日 + assert pd.weekend_days(date(2026, 10, 12), date(2026, 10, 14)) == 0 + assert pd.weekend_days(date(2026, 10, 1), date(2026, 10, 31)) == 9