mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-17 21:31:58 +00:00
分析算子的输入、目录事实、期次与现查:inputs.py(按名字取快照、拆「名字.r行.列」、列名不分大小写、遮罩列与非数值是调用错误);facts.py(按 SQL 认结果列取自哪张表的哪一列,读目录的度量类型、叫法、业务日期,只有确认过的度量类型才拿来判调用错误;查过的记进 params,复算不读目录);periods.py(日期、YYYY-MM、YYYY-Www、YYYY-Qn、整数序号,粒度按值推断);probe.py(make_prober 走 data/coverage 的只读探查:遮罩列不探、总时限、5 分钟缓存、退回剖析事实,另在库里算按运行当天现算的界;Book 先登记、查完再算,结论只记真用到的);data/coverage.py 加几个公开的小函数(parse_sql、as_day、literal_bound、window_of、column_range、bound_values、facts_range),_fallback 的读法抽成 _facts_range、行为不变;契约补几项可选字段(InputSnapshot 的 dialect / checks / no_data / mask_columns / masked_at_query,AnalysisInput 的 dialect,Probe 的 expr / value)
This commit is contained in:
@@ -188,6 +188,12 @@ class InputSnapshot(TypedDict, total=False):
|
||||
rows: list[list[Any]]
|
||||
truncated: bool
|
||||
masked: list[str] # 不能当维度、度量的列(遮罩列、按 SQL 认出改了名字的遮罩列),比较时不分大小写
|
||||
# 以下几项照查询快照里的原样带上(第 1 波 W1-a 加,都可选):
|
||||
dialect: str # SQL 的 sqlglot 方言(sqlite / mysql / postgres / oracle);不给时按探查器的 dialect(数据源名)
|
||||
checks: list[dict[str, Any]] # 快照里的 SQL 检查结果(level 为 error 的算可比性的数据问题)
|
||||
no_data: list[dict[str, Any]] # 快照里的「查询区间里这张表没有数据」
|
||||
mask_columns: list[str] # 没给 masked 时(复算取回的是快照工件本身)按这两项认遮罩列
|
||||
masked_at_query: list[str]
|
||||
|
||||
|
||||
class AnalysisInput(TypedDict, total=False):
|
||||
@@ -201,6 +207,7 @@ class AnalysisInput(TypedDict, total=False):
|
||||
row: int # 一格时:第几行、哪一列、值(十进制字符串)
|
||||
column: str
|
||||
value: str | None
|
||||
dialect: str # 读这份快照的 SQL 用的方言(第 1 波 W1-a 加,可选):复算按它读,界的写法才一样
|
||||
|
||||
|
||||
class Probe(TypedDict, total=False):
|
||||
@@ -213,6 +220,10 @@ class Probe(TypedDict, total=False):
|
||||
max: str | None
|
||||
basis: Literal["live", "profile", "observed"]
|
||||
at: str # 现查的时刻(ISO,服务端时区)
|
||||
# 按运行当天现算的界(CURRENT_DATE、date('now', …))在库里算出的值(第 1 波 W1-a 加,可选):有 expr 的这一条记的是界,
|
||||
# 没有 table / column / min / max
|
||||
expr: str # 界的 SQL 写法(按这份快照的方言)
|
||||
value: str | None # 库里算出的日期(带时刻的写成 YYYY-MM-DD HH:MM:SS)
|
||||
|
||||
|
||||
class Verdict(TypedDict, total=False):
|
||||
@@ -284,7 +295,7 @@ class Context(TypedDict, total=False):
|
||||
today: date # 运行当天(服务端时区):判本期完不完整
|
||||
probe: Prober | None # None 时只用 facts
|
||||
catalogs: Mapping[str, Any] # 数据源名 → 数据目录(app/data/catalog 的 notes):认度量类型、业务日期列、列的叫法
|
||||
facts: Mapping[str, Any] # 数据源名 → 剖析事实(探查失败时兜底)
|
||||
facts: Mapping[str, Any] # 数据源名 → 剖析事实(探查失败时兜底):{表名: last_facts 的那一项},或取它的函数
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
"""目录事实:结果列给人看的叫法、度量类型,表的业务日期(细案 01 §2.1、§6、§7.3)。
|
||||
|
||||
算子要知道「gmv 是流量还是比率」(比率当可加是调用错误;窗口不等时流量判不可比、其余只提醒)、「region 叫地区」(判定语
|
||||
写「分散在各地区」)、「orders 的业务日期是 order_date」(可比性检查按它取时间条件)。这些在数据目录里,可目录描述的是表的
|
||||
列,快照里的是 SQL 的结果列,所以先按 SQL 认结果列取自哪张表的哪一列,认法同单元格的显示规则(engine/cell_format:最外层
|
||||
SELECT 里按别名或列名找到那一项,看它是不是聚合、聚合的参数是不是一列、这一列在 FROM / JOIN 的哪张实表上),再查那一列
|
||||
的目录项:
|
||||
|
||||
- SUM(x)、不聚合的 x:x 的度量类型;COUNT(*) / COUNT(x):计数,按流量算;AVG(x):平均值,按比率算(不能按段相加);
|
||||
COUNT(DISTINCT x)、MIN / MAX、算式:认不出(当没标)。
|
||||
- 叫法:x 的中文名(目录里的 label);没有就是没有,调用方用列名。
|
||||
- 只认没被驳回的项(catalog.visible_notes)。度量类型另记有没有人确认过(confirmed / verified):只有确认过的才拿来判
|
||||
调用错误,推断的只写进附带说明(同 SQL 检查「宁可漏报,不可误报」)。
|
||||
|
||||
**复算不读目录**:run 时查过的都记进结论的 params["catalog"](record),复算时 Facts 从那里读,同一个键得到同一个答案;
|
||||
目录事后改了,封存的结论照样复算得出来。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
from app.data.names import name_key
|
||||
|
||||
#: 有确证的项状态:只有它们拿来判调用错误
|
||||
_STRONG = frozenset({"confirmed", "verified"})
|
||||
|
||||
|
||||
def _notes_of(entry: Any) -> Mapping[str, Any] | None:
|
||||
"""目录里一张表的 notes:CatalogEntry(有 .notes)或 notes 本身;去掉被驳回的项。"""
|
||||
from app.data import catalog
|
||||
|
||||
notes = getattr(entry, "notes", entry)
|
||||
return catalog.visible_notes(notes) if isinstance(notes, Mapping) else None
|
||||
|
||||
|
||||
def _table_notes(catalog: Mapping[str, Any] | None, table: str | None) -> Mapping[str, Any] | None:
|
||||
if not catalog or not table:
|
||||
return None
|
||||
wanted = {name_key(table), name_key(table.rsplit(".", 1)[-1])}
|
||||
for name, entry in catalog.items():
|
||||
if name_key(str(name)) in wanted or name_key(str(name).rsplit(".", 1)[-1]) in wanted:
|
||||
return _notes_of(entry)
|
||||
return None
|
||||
|
||||
|
||||
def _item(notes: Mapping[str, Any] | None, column: str, key: str) -> Mapping[str, Any] | None:
|
||||
columns = (notes or {}).get("columns") if isinstance(notes, Mapping) else None
|
||||
if not isinstance(columns, Mapping):
|
||||
return None
|
||||
for name, items in columns.items():
|
||||
if name_key(str(name)) == name_key(column) and isinstance(items, Mapping):
|
||||
item = items.get(key)
|
||||
return item if isinstance(item, Mapping) else None
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def _outer_select(sql: str) -> Any:
|
||||
"""最外层的 SELECT(engine/cell_format 的读法);同一条 SQL 只解析一次(只读,不改这棵树)。"""
|
||||
from app.engine import cell_format as cf
|
||||
|
||||
try:
|
||||
return cf._outer_select(sql)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
def source_of(sql: str, column: str) -> tuple[str | None, str | None, str | None, bool] | None:
|
||||
"""结果列 column 取自哪里:(表, 列, 聚合, 去重)。聚合是 sum / count / avg / min / max / other(别的聚合)或 None(不聚合);
|
||||
表在 SELECT * 、子查询、认不准的时候为 None。SQL 解析不了、结果列在最外层 SELECT 里找不到为 None。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.engine import cell_format as cf
|
||||
|
||||
try:
|
||||
select = _outer_select(sql)
|
||||
if select is None:
|
||||
return None
|
||||
item = cf._projection(select, column)
|
||||
if item is None:
|
||||
return None
|
||||
value = cf._unwrap(item.unalias())
|
||||
agg: str | None = None
|
||||
distinct = False
|
||||
if isinstance(value, exp.AggFunc):
|
||||
agg = {exp.Sum: "sum", exp.Count: "count", exp.Avg: "avg", exp.Min: "min", exp.Max: "max"}.get(
|
||||
type(value), "other")
|
||||
arg = value.this
|
||||
if isinstance(arg, exp.Distinct):
|
||||
distinct = True
|
||||
arg = arg.expressions[0] if len(arg.expressions) == 1 else None
|
||||
value = cf._unwrap(arg) if arg is not None else None
|
||||
if not isinstance(value, exp.Column) or isinstance(value.this, exp.Star):
|
||||
return None, None, agg, distinct
|
||||
tables = cf._tables(select)
|
||||
table = tables.get(name_key(value.table)) if value.table else \
|
||||
(next(iter(tables.values())) if len(tables) == 1 else None)
|
||||
return table, value.name, agg, distinct
|
||||
except Exception: # noqa: BLE001 - 认不出就当没标
|
||||
return None
|
||||
|
||||
|
||||
def date_source(sql: str, column: str) -> tuple[str, str] | None:
|
||||
"""期次结果列取自哪张表的哪个日期列:那一项选取里只引用了一列(order_date、DATE(order_date)、
|
||||
strftime('%Y-%m', order_date))时就是它。认不出为 None。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.engine import cell_format as cf
|
||||
|
||||
try:
|
||||
select = _outer_select(sql)
|
||||
item = cf._projection(select, column) if select is not None else None
|
||||
if item is None:
|
||||
return None
|
||||
cols = [c for c in item.unalias().find_all(exp.Column) if not isinstance(c.this, exp.Star)]
|
||||
if len({(name_key(c.table), name_key(c.name)) for c in cols}) != 1:
|
||||
return None
|
||||
ref = cols[0]
|
||||
tables = cf._tables(select)
|
||||
table = tables.get(name_key(ref.table)) if ref.table else \
|
||||
(next(iter(tables.values())) if len(tables) == 1 else None)
|
||||
return (table, ref.name) if table else None
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
class Facts:
|
||||
"""目录事实的取法:run 时 catalogs 是 context["catalogs"](数据源名 → {表名: notes 或 CatalogEntry}),复算时 recorded 是
|
||||
结论里记下的 params["catalog"]。查过的都记下(record),同一个键只认一次。"""
|
||||
|
||||
def __init__(self, catalogs: Mapping[str, Any] | None = None, recorded: Mapping[str, Any] | None = None) -> None:
|
||||
self._catalogs = catalogs or {}
|
||||
self._recorded = recorded
|
||||
self._labels: dict[str, str | None] = {}
|
||||
self._measures: dict[str, tuple[str | None, bool]] = {}
|
||||
self._dates: dict[str, str | None] = {}
|
||||
|
||||
def _catalog(self, source: str) -> Mapping[str, Any] | None:
|
||||
found = self._catalogs.get(source)
|
||||
return found if isinstance(found, Mapping) else None
|
||||
|
||||
def label(self, snap: Any, column: str) -> str | None:
|
||||
"""结果列给人看的叫法(目录里取自的那一列的中文名);没有为 None。"""
|
||||
if column in self._labels:
|
||||
return self._labels[column]
|
||||
if self._recorded is not None:
|
||||
value = (self._recorded.get("labels") or {}).get(column)
|
||||
else:
|
||||
value = None
|
||||
src = source_of(snap.sql, column)
|
||||
if src is not None and src[1] is not None:
|
||||
item = _item(_table_notes(self._catalog(snap.source), src[0]), src[1], "label")
|
||||
text = item.get("value") if item else None
|
||||
value = text.strip() if isinstance(text, str) and text.strip() else None
|
||||
self._labels[column] = value
|
||||
return value
|
||||
|
||||
def measure(self, snap: Any, column: str) -> tuple[str | None, bool]:
|
||||
"""结果列的度量类型和有没有人确认过:(flow / stock / ratio / identifier / status / attribute 或 None, 确认过)。"""
|
||||
if column in self._measures:
|
||||
return self._measures[column]
|
||||
if self._recorded is not None:
|
||||
kind = (self._recorded.get("measures") or {}).get(column)
|
||||
out = (kind, kind is not None and column not in (self._recorded.get("unconfirmed") or []))
|
||||
else:
|
||||
out = self._live_measure(snap, column)
|
||||
self._measures[column] = out
|
||||
return out
|
||||
|
||||
def _live_measure(self, snap: Any, column: str) -> tuple[str | None, bool]:
|
||||
src = source_of(snap.sql, column)
|
||||
if src is None:
|
||||
return None, False
|
||||
table, name, agg, distinct = src
|
||||
if agg == "count" and not distinct:
|
||||
return "flow", True
|
||||
if agg == "avg":
|
||||
return "ratio", True
|
||||
if agg not in (None, "sum") or name is None:
|
||||
return None, False
|
||||
item = _item(_table_notes(self._catalog(snap.source), table), name, "measure")
|
||||
kind = item.get("value") if item else None
|
||||
if not isinstance(kind, str):
|
||||
return None, False
|
||||
return kind, item.get("status") in _STRONG
|
||||
|
||||
def business_date(self, source: str, table: str) -> str | None:
|
||||
"""表的业务日期列(目录的 business_date);没写为 None。"""
|
||||
key = name_key(table)
|
||||
if key in self._dates:
|
||||
return self._dates[key]
|
||||
if self._recorded is not None:
|
||||
value = (self._recorded.get("dates") or {}).get(key)
|
||||
else:
|
||||
notes = _table_notes(self._catalog(source), table)
|
||||
item = notes.get("business_date") if isinstance(notes, Mapping) else None
|
||||
raw = item.get("value") if isinstance(item, Mapping) else None
|
||||
col = raw.get("column") if isinstance(raw, Mapping) else None
|
||||
value = col if isinstance(col, str) and col.strip() else None
|
||||
self._dates[key] = value
|
||||
return value
|
||||
|
||||
def record(self) -> dict[str, Any] | None:
|
||||
"""查过、查得到的事实(写进 params["catalog"]);一项都没有为 None。"""
|
||||
out: dict[str, Any] = {}
|
||||
labels = {k: v for k, v in self._labels.items() if v is not None}
|
||||
measures = {k: v[0] for k, v in self._measures.items() if v[0] is not None}
|
||||
unconfirmed = sorted(k for k, v in self._measures.items() if v[0] is not None and not v[1])
|
||||
dates = {k: v for k, v in self._dates.items() if v is not None}
|
||||
for key, value in (("labels", labels), ("measures", measures), ("unconfirmed", unconfirmed), ("dates", dates)):
|
||||
if value:
|
||||
out[key] = value
|
||||
return out or None
|
||||
|
||||
|
||||
__all__ = ["Facts", "date_source", "source_of"]
|
||||
@@ -0,0 +1,220 @@
|
||||
"""算子的输入:按名字取快照、拆单元格的写法、按列名认列、遮罩列、度量格换成精确的数、同一键同一期多行(细案 01 §1、§8.1)。
|
||||
|
||||
- 名字:agent 写本节点查过的编号(Q3,和交回模型的「【证据 Q3】」同一个),分析卡写来源节点 id;resolve 取不到就是调用错误。
|
||||
- 单元格写成「名字.r行.列」(Q1.r0.gmv),行从 0 数;名字里可以有点(取最后一个「.r数字.」切开)。
|
||||
- 列名比较不分大小写,按快照的 columns 定位:一字不差的优先;只差大小写的有好几列时是调用错误。
|
||||
- 遮罩列(快照的 masked,不分大小写)不能当维度、度量、期次:调用错误。结论里只出现非遮罩列的格。
|
||||
- 度量格:数(含 DECIMAL 落成的数字文本)换成 Fraction;空值为 None(各算子按细案 §1 处理);文本、布尔是调用错误。
|
||||
查询时记了类型的列按类型认:text 列里的 "2026" 不当数。
|
||||
|
||||
调用错误(contract.CallError)的说法一种,写给 agent:说清哪里不对、怎么改。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
from app.analysis.contract import AnalysisInput, CallError, InputSnapshot
|
||||
from app.analysis.numbers import canon, to_fraction
|
||||
|
||||
#: 单元格的写法:名字.r行.列
|
||||
_CELL = re.compile(r"^(?P<name>.+)\.r(?P<row>\d+)\.(?P<column>.+)$")
|
||||
#: 报错里列出的列最多几个
|
||||
_LIST_MAX = 12
|
||||
|
||||
|
||||
def _listed(columns: Sequence[str]) -> str:
|
||||
shown = "、".join(columns[:_LIST_MAX])
|
||||
return shown + (f"等 {len(columns)} 列" if len(columns) > _LIST_MAX else "")
|
||||
|
||||
|
||||
def hashable(value: Any) -> Any:
|
||||
"""维度值当键:文本、数、布尔、空值原样;别的(列表、对象)和非数、无穷写成文字(同 engine/compare._hashable)。"""
|
||||
if isinstance(value, float) and not math.isfinite(value):
|
||||
return repr(value)
|
||||
return value if isinstance(value, (str, int, float, bool)) or value is None else repr(value)
|
||||
|
||||
|
||||
def same_value(a: Any, b: Any) -> bool:
|
||||
"""调用里写的期次、段和快照里的值是不是同一个:相等,或者写出来一样("2026" 和 2026、" 华东" 和 "华东")。"""
|
||||
if a == b and type(a) is type(b):
|
||||
return True
|
||||
if isinstance(a, bool) or isinstance(b, bool):
|
||||
return a == b
|
||||
fa, fb = to_fraction(a), to_fraction(b)
|
||||
if fa is not None and fb is not None and not isinstance(a, str) and not isinstance(b, str):
|
||||
return fa == fb
|
||||
return str(a).strip() == str(b).strip()
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class Snap:
|
||||
"""一份输入快照(InputSnapshot 的薄包装)。"""
|
||||
|
||||
ref: str
|
||||
data: Mapping[str, Any]
|
||||
dialect: str | None = None
|
||||
columns: list[str] = field(default_factory=list)
|
||||
rows: list[list[Any]] = field(default_factory=list)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.columns = [str(c) for c in self.data.get("columns") or []]
|
||||
self.rows = [list(r) if isinstance(r, (list, tuple)) else [] for r in self.data.get("rows") or []]
|
||||
|
||||
@property
|
||||
def artifact(self) -> str | None:
|
||||
value = self.data.get("artifact")
|
||||
return value if isinstance(value, str) and value else None
|
||||
|
||||
@property
|
||||
def source(self) -> str:
|
||||
return str(self.data.get("source") or "")
|
||||
|
||||
@property
|
||||
def sql(self) -> str:
|
||||
return str(self.data.get("sql") or "")
|
||||
|
||||
@property
|
||||
def truncated(self) -> bool:
|
||||
return bool(self.data.get("truncated"))
|
||||
|
||||
@property
|
||||
def masked(self) -> set[str]:
|
||||
"""不能当维度、度量的列(小写):入口交来的 masked;没交(复算时取回的是快照工件本身)就按快照记下的遮罩列
|
||||
(数据源设的 mask_columns、查询时整列遮掉的 masked_at_query)。"""
|
||||
found = self.data.get("masked")
|
||||
if found is None:
|
||||
found = [*(self.data.get("mask_columns") or []), *(self.data.get("masked_at_query") or [])]
|
||||
return {str(m).strip().lower() for m in found or [] if str(m).strip()}
|
||||
|
||||
def kind(self, column: str) -> str | None:
|
||||
"""查询时记下的这一列的类型(column_types 是 {列: 类型} 或按列序的列表)。"""
|
||||
types = self.data.get("column_types")
|
||||
if isinstance(types, Mapping):
|
||||
kind = types.get(column)
|
||||
elif isinstance(types, list) and column in self.columns:
|
||||
i = self.columns.index(column)
|
||||
kind = types[i] if i < len(types) else None
|
||||
else:
|
||||
kind = None
|
||||
return kind if isinstance(kind, str) else None
|
||||
|
||||
def column(self, wanted: Any, role: str) -> str:
|
||||
"""按名字认一列(不分大小写),认出的是快照里的写法。不存在、只差大小写的好几列、遮罩列都是调用错误。"""
|
||||
if not isinstance(wanted, str) or not wanted.strip():
|
||||
raise CallError("missing_column", f"没写{role}:写 {self.ref} 里的列名")
|
||||
name = wanted.strip()
|
||||
hits = [c for c in self.columns if c == name] or [c for c in self.columns if c.lower() == name.lower()]
|
||||
if not hits:
|
||||
raise CallError("unknown_column", f"{self.ref} 里没有「{name}」这一列,{role}要写它的列名:{self.ref} 的列有 "
|
||||
f"{_listed(self.columns)}")
|
||||
if len(hits) > 1:
|
||||
raise CallError("ambiguous_column", f"{self.ref} 里有好几列都叫「{name}」(只差大小写):{role}写出确切的列名")
|
||||
column = hits[0]
|
||||
if column.lower() in self.masked:
|
||||
raise CallError("masked_column", f"「{column}」是遮罩列,不能当{role}:换一列,或者在 SQL 里不选它")
|
||||
return column
|
||||
|
||||
def index(self, column: str) -> int:
|
||||
return self.columns.index(column)
|
||||
|
||||
def cell(self, row: int, column: str) -> Any:
|
||||
r = self.rows[row]
|
||||
i = self.index(column)
|
||||
return r[i] if i < len(r) else None
|
||||
|
||||
def number(self, row: int, column: str, role: str) -> Fraction | None:
|
||||
"""一格换成精确的数;空值为 None。不是数是调用错误。"""
|
||||
raw = self.cell(row, column)
|
||||
if raw is None:
|
||||
return None
|
||||
if isinstance(raw, bool) or (isinstance(raw, str) and self.kind(column) == "text"):
|
||||
raise self._not_numeric(row, column, raw, role)
|
||||
if isinstance(raw, str):
|
||||
from app.engine.expressions import numeric_text
|
||||
|
||||
if numeric_text(raw.strip()) is None:
|
||||
raise self._not_numeric(row, column, raw, role)
|
||||
value = to_fraction(raw)
|
||||
if value is None:
|
||||
raise self._not_numeric(row, column, raw, role)
|
||||
return value
|
||||
|
||||
def _not_numeric(self, row: int, column: str, raw: Any, role: str) -> CallError:
|
||||
shown = str(raw).replace("\n", " ")[:20]
|
||||
return CallError("not_numeric", f"{self.ref} 的「{column}」不是数值列(第 {row} 行是「{shown}」):{role}要选数值列")
|
||||
|
||||
|
||||
class Inputs:
|
||||
"""按名字取输入,记下结论的 inputs(同一份快照、同一格只记一次,按用到的先后)。"""
|
||||
|
||||
def __init__(self, resolve: Callable[[str], InputSnapshot | None],
|
||||
dialect: Callable[[str, Mapping[str, Any]], str | None] | None = None) -> None:
|
||||
self._resolve = resolve
|
||||
self._dialect = dialect
|
||||
self._snaps: dict[str, Snap] = {}
|
||||
self.records: list[AnalysisInput] = []
|
||||
|
||||
def snap(self, name: Any, role: str) -> Snap:
|
||||
"""按名字取一份快照。"""
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise CallError("missing_query", f"没写{role}:写本节点查过的查询编号(如 Q3)")
|
||||
key = name.strip()
|
||||
if key not in self._snaps:
|
||||
found = self._resolve(key)
|
||||
if not isinstance(found, Mapping):
|
||||
raise CallError("unknown_query", f"没有叫「{key}」的查询结果:{role}写本节点查过的查询编号(如 Q3)")
|
||||
self._snaps[key] = Snap(key, found, self._dialect(key, found) if self._dialect else found.get("dialect"))
|
||||
return self._snaps[key]
|
||||
|
||||
def cell(self, ref: Any, role: str, label: str) -> tuple[Snap, int, str, Fraction | None]:
|
||||
"""「名字.r行.列」→ (快照, 行, 列, 值)。"""
|
||||
found = _CELL.match(ref.strip()) if isinstance(ref, str) else None
|
||||
if found is None:
|
||||
raise CallError("bad_cell", f"{label}要写成单元格「编号.r行.列」(如 Q1.r0.gmv),现在是「{ref}」")
|
||||
snap = self.snap(found.group("name"), label)
|
||||
row = int(found.group("row"))
|
||||
if row >= len(snap.rows):
|
||||
raise CallError("row_out_of_range", f"{snap.ref} 没有第 {row} 行(共 {len(snap.rows)} 行,从 r0 数起):改{label}")
|
||||
column = snap.column(found.group("column"), label)
|
||||
value = snap.number(row, column, label)
|
||||
self.record({"ref": f"{snap.ref}.r{row}.{column}", "role": role, "row": row, "column": column,
|
||||
"value": canon(value)}, snap)
|
||||
return snap, row, column, value
|
||||
|
||||
def use(self, snap: Snap, role: str, columns: Sequence[str], rows: Sequence[int] | None = None) -> None:
|
||||
"""记下一份快照的用法:用了哪几列、哪几行(全用了写 all)。"""
|
||||
used = sorted(set(rows)) if rows is not None else None
|
||||
shown: Any = "all" if used is None or used == list(range(len(snap.rows))) else used
|
||||
self.record({"ref": snap.ref, "role": role, "columns": list(dict.fromkeys(columns)), "rows": shown}, snap)
|
||||
|
||||
def record(self, entry: dict[str, Any], snap: Snap) -> None:
|
||||
out: dict[str, Any] = {"ref": entry["ref"]}
|
||||
if snap.artifact:
|
||||
out["artifact"] = snap.artifact
|
||||
if snap.dialect:
|
||||
out["dialect"] = snap.dialect
|
||||
out.update({k: v for k, v in entry.items() if k != "ref"})
|
||||
if out not in self.records:
|
||||
self.records.append(out) # type: ignore[arg-type]
|
||||
|
||||
def parent(self, name: str, artifact: str) -> None:
|
||||
entry: dict[str, Any] = {"ref": name, "artifact": artifact, "role": "parent"}
|
||||
if entry not in self.records:
|
||||
self.records.append(entry) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def as_list(value: Any) -> list[Any]:
|
||||
"""参数里一个值或一组值:都规范成列表(None 是空列表)。"""
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, (list, tuple)):
|
||||
return list(value)
|
||||
return [value]
|
||||
|
||||
|
||||
__all__ = ["Inputs", "Snap", "as_list", "hashable", "same_value"]
|
||||
@@ -0,0 +1,224 @@
|
||||
"""期次:快照里期次列的值认成可以排序、能判断连不连续、知道覆盖哪几天的期(细案 01 §3「期次认得出、能排序」)。
|
||||
|
||||
认这几种写法(一列里要是同一种):
|
||||
|
||||
- 日期:DATE / DATETIME(零点)、'YYYY-MM-DD'、'YYYYMMDD'、8 位整数。粒度不写时按值推断:都是某月 1 日(或都是月末)
|
||||
且相邻两期隔整月 → month(月份都在 1、4、7、10 月且隔整季 → quarter);相邻两期都隔 7 天的整数倍 → week(这个日期当
|
||||
一周的第一天);否则 day。
|
||||
- 月份:'YYYY-MM'、'YYYYMM'(6 位,月份 01–12)→ month。
|
||||
- ISO 周:'YYYY-Www'(2026-W42 是 2026-10-12 到 10-18)→ week。SQLite 的 strftime('%W') 不是 ISO 周号(年初那几天
|
||||
算第 0 周),按它写的期次覆盖的日子会差一周:方法里要求期次写成每周第一天的日期。
|
||||
- 季度:'YYYY-Qn'、'YYYYQn' → quarter。
|
||||
- 整数序号(不是上面几种的整数)→ seq:能排序、能判连续,不知道覆盖哪几天(本期完不完整核对不了)。
|
||||
|
||||
带时刻(非零点)的、写法混着的、认不出的都算认不出(period_unreadable)。
|
||||
|
||||
每一期记 ordinal:同一粒度里相邻两期的 ordinal 差 1(日:天数,周:从第一期起第几个 7 天,月:年 × 12 + 月,季:年 × 4 +
|
||||
季,序号:本身),据此排序、判缺期、按 lag 找比较期。start / end 是这一期覆盖的第一天和最后一天(含),seq 没有。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import calendar
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from app.data import coverage
|
||||
|
||||
GRAINS = ("day", "week", "month", "quarter")
|
||||
#: 粒度的叫法:数量词(过去 12 周、12 个月)、每…(每周、每月)
|
||||
COUNT_WORD = {"day": "天", "week": "周", "month": "个月", "quarter": "个季度", "seq": "期"}
|
||||
PER_WORD = {"day": "天", "week": "周", "month": "月", "quarter": "季度", "seq": "期"}
|
||||
#: 粒度不写时各粒度的缺省 lag:日数据和上周同一天比(避开星期效应),其余和上一期比
|
||||
DEFAULT_LAG = {"day": 7, "week": 1, "month": 1, "quarter": 1, "seq": 1}
|
||||
|
||||
_MONTH = re.compile(r"^(\d{4})-?(\d{2})$")
|
||||
_WEEK = re.compile(r"^(\d{4})-?W(\d{1,2})$", re.I)
|
||||
_QUARTER = re.compile(r"^(\d{4})-?Q([1-4])$", re.I)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Period:
|
||||
raw: Any # 快照里的原值
|
||||
ordinal: int
|
||||
start: date | None
|
||||
end: date | None
|
||||
|
||||
|
||||
def _month_end(d: date) -> date:
|
||||
return d.replace(day=calendar.monthrange(d.year, d.month)[1])
|
||||
|
||||
|
||||
def _quarter_start(d: date) -> date:
|
||||
return date(d.year, 3 * ((d.month - 1) // 3) + 1, 1)
|
||||
|
||||
|
||||
def _quarter_end(d: date) -> date:
|
||||
start = _quarter_start(d)
|
||||
return _month_end(date(start.year, start.month + 2, 1))
|
||||
|
||||
|
||||
def _as_date(value: Any) -> date | None:
|
||||
"""日期写法的期次 → 那一天;带非零点时刻的、认不出的为 None。"""
|
||||
if isinstance(value, datetime):
|
||||
return value.date() if value.time() == datetime.min.time() else None
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
day = coverage.as_day(value)
|
||||
if day is None or day.timed:
|
||||
return None
|
||||
return day.day
|
||||
|
||||
|
||||
def _shape(value: Any) -> str | None:
|
||||
"""一个期次值是哪种写法:date / month / week / quarter / seq;认不出为 None。"""
|
||||
if isinstance(value, bool) or value is None:
|
||||
return None
|
||||
if isinstance(value, (date, datetime)):
|
||||
return "date" if _as_date(value) else None
|
||||
if isinstance(value, float) and value.is_integer():
|
||||
value = int(value)
|
||||
if isinstance(value, int):
|
||||
text = str(value)
|
||||
if len(text) == 8 and _as_date(value):
|
||||
return "date"
|
||||
if len(text) == 6 and (m := _MONTH.match(text)) and 1 <= int(m.group(2)) <= 12:
|
||||
return "month"
|
||||
return "seq"
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
text = value.strip()
|
||||
if (m := _MONTH.match(text)) and 1 <= int(m.group(2)) <= 12 and 1900 <= int(m.group(1)) <= 2200:
|
||||
return "month"
|
||||
if _WEEK.match(text):
|
||||
return "week"
|
||||
if _QUARTER.match(text):
|
||||
return "quarter"
|
||||
if _as_date(text):
|
||||
return "date"
|
||||
if re.fullmatch(r"-?\d+", text):
|
||||
return "seq"
|
||||
return None
|
||||
|
||||
|
||||
def _month_of(value: Any) -> date:
|
||||
m = _MONTH.match(str(int(value) if isinstance(value, (int, float)) else value).strip())
|
||||
assert m is not None
|
||||
return date(int(m.group(1)), int(m.group(2)), 1)
|
||||
|
||||
|
||||
def _infer_date_grain(days: Sequence[date]) -> str:
|
||||
"""一列日期的粒度(见文件头)。"""
|
||||
ordered = sorted(set(days))
|
||||
if len(ordered) < 2:
|
||||
return "day"
|
||||
firsts = all(d.day == 1 for d in ordered)
|
||||
lasts = all(d == _month_end(d) for d in ordered)
|
||||
if firsts or lasts:
|
||||
months = [d.year * 12 + d.month - 1 for d in ordered]
|
||||
gaps = [b - a for a, b in zip(months, months[1:])]
|
||||
if all(g >= 1 for g in gaps):
|
||||
if all(d.month in (1, 4, 7, 10) for d in ordered) and all(g % 3 == 0 for g in gaps) and firsts:
|
||||
return "quarter"
|
||||
return "month"
|
||||
gaps = [(b - a).days for a, b in zip(ordered, ordered[1:])]
|
||||
if all(g % 7 == 0 for g in gaps):
|
||||
return "week"
|
||||
return "day"
|
||||
|
||||
|
||||
def parse(values: Sequence[Any], grain: str | None = None) -> tuple[list[Period], str] | None:
|
||||
"""一列期次值 → (逐个的 Period,粒度)。认不出(写法不一、带时刻、粒度和写法对不上)返回 None。"""
|
||||
shapes = {_shape(v) for v in values}
|
||||
if not values or None in shapes or len(shapes) != 1:
|
||||
return None
|
||||
shape = shapes.pop()
|
||||
if shape == "seq":
|
||||
if grain not in (None, "seq"):
|
||||
return None
|
||||
return [Period(v, int(v), None, None) for v in values], "seq"
|
||||
if shape == "month":
|
||||
if grain not in (None, "month"):
|
||||
return None
|
||||
out = []
|
||||
for v in values:
|
||||
start = _month_of(v)
|
||||
out.append(Period(v, start.year * 12 + start.month - 1, start, _month_end(start)))
|
||||
return out, "month"
|
||||
if shape == "week":
|
||||
if grain not in (None, "week"):
|
||||
return None
|
||||
out = []
|
||||
for v in values:
|
||||
m = _WEEK.match(str(v).strip())
|
||||
assert m is not None
|
||||
try:
|
||||
start = date.fromisocalendar(int(m.group(1)), int(m.group(2)), 1)
|
||||
except ValueError:
|
||||
return None
|
||||
out.append(Period(v, start.toordinal() // 7, start, start + timedelta(days=6)))
|
||||
return out, "week"
|
||||
if shape == "quarter":
|
||||
if grain not in (None, "quarter"):
|
||||
return None
|
||||
out = []
|
||||
for v in values:
|
||||
m = _QUARTER.match(str(v).strip())
|
||||
assert m is not None
|
||||
start = date(int(m.group(1)), 3 * (int(m.group(2)) - 1) + 1, 1)
|
||||
out.append(Period(v, start.year * 4 + (start.month - 1) // 3, start, _quarter_end(start)))
|
||||
return out, "quarter"
|
||||
days = [_as_date(v) for v in values]
|
||||
assert all(days)
|
||||
grain = grain or _infer_date_grain(days) # type: ignore[arg-type]
|
||||
if grain == "day":
|
||||
return [Period(v, d.toordinal(), d, d) for v, d in zip(values, days)], "day" # type: ignore[union-attr]
|
||||
if grain == "week":
|
||||
first = min(days) # type: ignore[type-var]
|
||||
if any((d - first).days % 7 for d in days): # type: ignore[operator]
|
||||
return None
|
||||
return [Period(v, (d - first).days // 7, d, d + timedelta(days=6)) # type: ignore[operator]
|
||||
for v, d in zip(values, days)], "week"
|
||||
if grain == "month":
|
||||
if not all(d.day == 1 or d == _month_end(d) for d in days): # type: ignore[union-attr]
|
||||
return None
|
||||
return [Period(v, d.year * 12 + d.month - 1, d.replace(day=1), _month_end(d)) # type: ignore[union-attr]
|
||||
for v, d in zip(values, days)], "month"
|
||||
if grain == "quarter":
|
||||
if not all(d.day == 1 and d.month in (1, 4, 7, 10) for d in days): # type: ignore[union-attr]
|
||||
return None
|
||||
return [Period(v, d.year * 4 + (d.month - 1) // 3, d, _quarter_end(d)) # type: ignore[union-attr]
|
||||
for v, d in zip(values, days)], "quarter"
|
||||
return None
|
||||
|
||||
|
||||
def calendar_unit(start: date, end: date) -> str | None:
|
||||
"""这段日子是不是一个整的日历期:整月 month、整季 quarter、整年 year;都不是为 None。整月对整月算一样长(31 天的 10 月
|
||||
和 30 天的 9 月环比是通行的比法),不按天数判窗口不等。"""
|
||||
if start.day != 1:
|
||||
return None
|
||||
if start.month == 1 and end == date(start.year, 12, 31):
|
||||
return "year"
|
||||
if start.month in (1, 4, 7, 10) and end == _quarter_end(start):
|
||||
return "quarter"
|
||||
if end == _month_end(start):
|
||||
return "month"
|
||||
return None
|
||||
|
||||
|
||||
def weekend_days(start: date, end: date) -> int:
|
||||
"""[start, end] 里周六、周日的天数。"""
|
||||
days = (end - start).days + 1
|
||||
full, rest = divmod(days, 7)
|
||||
count = 2 * full
|
||||
for k in range(rest):
|
||||
if (start + timedelta(days=7 * full + k)).weekday() >= 5:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
__all__ = ["COUNT_WORD", "DEFAULT_LAG", "GRAINS", "PER_WORD", "Period", "calendar_unit", "parse", "weekend_days"]
|
||||
@@ -0,0 +1,254 @@
|
||||
"""可比性检查的现查(细案 01 §1「只读快照、不发查询」的唯一例外,§6):表里这一列的实际日期范围、按运行当天现算的界。
|
||||
|
||||
算子本身不查库。要现查的只有两样,都走 data/coverage 现成的只读探查(只读守卫、遮罩列不探、每条 PROBE_TIMEOUT_S 秒、总
|
||||
时限、同一张表 5 分钟内不重查、现查不成退回剖析事实):
|
||||
|
||||
- 范围:(数据源名, 表, 列) → contract.Probe {source, table, column, min, max, basis, at},min / max 是 ISO 日期;表里这一列
|
||||
一个值都没有时 min、max 都为 None(basis live)。basis 是 live(现查)/ profile(按剖析)/ observed(按取值观察),at 是现查的
|
||||
时刻(服务端时区,ISO,秒)或剖析、观察的日期。
|
||||
- 界:SQL 里按运行当天现算的界(CURRENT_DATE、DATE_SUB(…)、date('now', …))在同一个库里算出来:(数据源名, 界的 SQL 写法)
|
||||
→ Probe {source, expr, value, basis: live, at},value 是库里算出的日期(带时刻的写成「YYYY-MM-DD HH:MM:SS」)。
|
||||
|
||||
**结论里记下用到的现查结果,复算用记下的值**:Book 是一次分析的现查簿。run 时算子第一遍只登记要查什么(拿到的都是「不
|
||||
知道」),入口查完再算一遍,直到不再要新的(算子在拿到范围之后才可能要界,最多三轮);复算时 Book 里装的是结论记下的
|
||||
probes,不现查。结论的 probes 只记最后一遍真用到、查得到的,按用到的先后——同样的输入、同样的现查结果,结论逐字节相同。
|
||||
|
||||
入口(agent 分析工具、分析卡、口径卡的可比性挂钩)用 make_prober 生成 contract.Prober 交进 context["probe"],见它的说明。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from app.analysis.contract import Probe
|
||||
from app.data.names import name_key
|
||||
|
||||
#: 算子在拿到现查结果之后可能还要新的(先要范围、再要界):run 最多这么多轮
|
||||
MAX_ROUNDS = 3
|
||||
|
||||
|
||||
def _range_key(source: str, table: str, column: str) -> tuple[str, str, str, str]:
|
||||
return ("range", source or "", name_key(str(table)), name_key(str(column)))
|
||||
|
||||
|
||||
def _bound_key(source: str, expr: str) -> tuple[str, str, str]:
|
||||
return ("bound", source or "", str(expr))
|
||||
|
||||
|
||||
def probe_key(probe: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
"""一条记下的现查结果的键(Book 按它查)。"""
|
||||
if probe.get("expr") is not None:
|
||||
return _bound_key(str(probe.get("source") or ""), str(probe["expr"]))
|
||||
return _range_key(str(probe.get("source") or ""), str(probe.get("table") or ""), str(probe.get("column") or ""))
|
||||
|
||||
|
||||
class Book:
|
||||
"""一次分析的现查簿。known:{键: Probe 或 None(查过、查不到)}。算子经 range / bound 取;asked 是按先后登记的键,missing
|
||||
是登记了、簿里还没有的(run 据此去查),used 是最后真用到、查得到的(写进结论的 probes)。"""
|
||||
|
||||
def __init__(self, known: Mapping[tuple[str, ...], Probe | None] | Iterable[Probe] = ()) -> None:
|
||||
if isinstance(known, Mapping):
|
||||
self._known: dict[tuple[str, ...], Probe | None] = dict(known)
|
||||
self._closed = False
|
||||
else:
|
||||
self._known = {probe_key(p): p for p in known}
|
||||
self._closed = True # 复算:记下的就是全部,没记的当查不到
|
||||
self.asked: list[tuple[str, ...]] = []
|
||||
|
||||
def _ask(self, key: tuple[str, ...]) -> Probe | None:
|
||||
if key not in self.asked:
|
||||
self.asked.append(key)
|
||||
return self._known.get(key)
|
||||
|
||||
def range(self, source: str, table: str, column: str) -> Probe | None:
|
||||
return self._ask(_range_key(source, table, column))
|
||||
|
||||
def bound(self, source: str, expr: str) -> Probe | None:
|
||||
return self._ask(_bound_key(source, expr))
|
||||
|
||||
@property
|
||||
def missing(self) -> list[tuple[str, ...]]:
|
||||
return [] if self._closed else [k for k in self.asked if k not in self._known]
|
||||
|
||||
@property
|
||||
def used(self) -> list[Probe]:
|
||||
return [p for p in (self._known.get(k) for k in self.asked) if p is not None]
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now().astimezone().replace(tzinfo=None).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
async def fetch(book: Book, context: Mapping[str, Any]) -> dict[tuple[str, ...], Probe | None]:
|
||||
"""把簿里登记了、还没有的现查一遍:先 context["probe"],查不到的范围退回 context["facts"](数据源名 → {表名: 剖析事实},
|
||||
或数据源名 → 取剖析事实的函数),都没有记 None。交回新查到的那几条。"""
|
||||
from app.data import coverage
|
||||
|
||||
prober = context.get("probe")
|
||||
facts = context.get("facts") or {}
|
||||
out: dict[tuple[str, ...], Probe | None] = {}
|
||||
for key in book.missing:
|
||||
found: Probe | None = None
|
||||
if key[0] == "range":
|
||||
_, source, table, column = key
|
||||
if prober is not None:
|
||||
try:
|
||||
found = await prober(source, table, column)
|
||||
except Exception: # noqa: BLE001 - 探查失败就当不知道
|
||||
found = None
|
||||
if found is None:
|
||||
found = await _from_facts(facts.get(source), source, table, column, coverage)
|
||||
else:
|
||||
_, source, expr = key
|
||||
bounds = getattr(prober, "bounds", None)
|
||||
if bounds is not None:
|
||||
try:
|
||||
found = await bounds(source, expr)
|
||||
except Exception: # noqa: BLE001
|
||||
found = None
|
||||
out[key] = found
|
||||
return out
|
||||
|
||||
|
||||
async def _from_facts(facts: Any, source: str, table: str, column: str, coverage: Any) -> Probe | None:
|
||||
"""探查不成时按剖析事实兜底:facts 是 {表名: last_facts 的那一项} 或取它的函数(coverage.Facts)。"""
|
||||
found: Any = None
|
||||
try:
|
||||
if callable(facts):
|
||||
found = await facts(table)
|
||||
elif isinstance(facts, Mapping):
|
||||
found = next((v for k, v in facts.items() if name_key(str(k)) == name_key(table)
|
||||
or name_key(str(k).rsplit(".", 1)[-1]) == name_key(table.rsplit(".", 1)[-1])), None)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
data = coverage.facts_range(found, column)
|
||||
return _range_probe(source, table, column, data) if data else None
|
||||
|
||||
|
||||
def _range_probe(source: str, table: str, column: str, data: Mapping[str, Any]) -> Probe:
|
||||
basis = data.get("basis") if data.get("basis") in ("live", "profile", "observed") else "live"
|
||||
at = _now() if basis == "live" else str(data.get("at") or "")
|
||||
return {"source": source, "table": table, "column": column, "min": data.get("min"), "max": data.get("max"),
|
||||
"basis": basis, "at": at} # type: ignore[typeddict-item]
|
||||
|
||||
|
||||
class LiveProber:
|
||||
"""contract.Prober:await prober(数据源名, 表, 列) → Probe | None;另有 await prober.bounds(数据源名, 界的 SQL) 在库里算界,
|
||||
prober.dialect(数据源名) 是这个源的 sqlglot 方言(算子按它读 SQL、写界)。见 make_prober。"""
|
||||
|
||||
def __init__(self, sources: Mapping[str, Any], *, masked: Mapping[str, Iterable[str]] | None,
|
||||
facts: Mapping[str, Callable[[str], Awaitable[Mapping[str, Any] | None]]] | None,
|
||||
cache: Mapping[str, Any] | None, budget: float,
|
||||
clock: Callable[[], str] | None) -> None:
|
||||
from app.data import coverage
|
||||
|
||||
self._sources = dict(sources)
|
||||
self._masked = {k: {str(c).strip().lower() for c in v} for k, v in (masked or {}).items()}
|
||||
self._facts = dict(facts or {})
|
||||
self._cache: dict[str, Any] = dict(cache or {})
|
||||
self._budget = budget
|
||||
self._deadline: float | None = None
|
||||
self._clock = clock or _now
|
||||
self._bounds: dict[tuple[str, str], Any] = {}
|
||||
self._coverage = coverage
|
||||
|
||||
def dialect(self, source: str) -> str | None:
|
||||
from app.data.sqlcheck import dialect_of
|
||||
|
||||
found = self._sources.get(source)
|
||||
return dialect_of(getattr(found, "kind", None)) if found is not None else None
|
||||
|
||||
def _expired(self) -> bool:
|
||||
if self._deadline is None:
|
||||
self._deadline = time.monotonic() + self._budget
|
||||
return time.monotonic() > self._deadline
|
||||
|
||||
def _hidden(self, source: Any, name: str) -> set[str]:
|
||||
if name in self._masked:
|
||||
return self._masked[name]
|
||||
from app.data.engine import masked_columns
|
||||
|
||||
return {c.lower() for c in masked_columns(getattr(source, "options", None) or {})}
|
||||
|
||||
def _facts_of(self, name: str, source: Any) -> Callable[[str], Awaitable[Mapping[str, Any] | None]]:
|
||||
return self._facts.get(name) or profile_facts(source)
|
||||
|
||||
async def __call__(self, source: str, table: str, column: str) -> Probe | None:
|
||||
found = self._sources.get(source)
|
||||
if found is None or str(column).strip().lower() in self._hidden(found, source):
|
||||
return None
|
||||
if self._expired():
|
||||
return None
|
||||
cache = self._cache.setdefault(source, self._coverage.Cache())
|
||||
data = await self._coverage.column_range(found, table, column, facts=self._facts_of(source, found), cache=cache)
|
||||
if data is None:
|
||||
return None
|
||||
probe = _range_probe(source, table, column, data)
|
||||
if probe["basis"] == "live":
|
||||
probe["at"] = self._clock()
|
||||
return probe
|
||||
|
||||
async def bounds(self, source: str, expr: str) -> Probe | None:
|
||||
found = self._sources.get(source)
|
||||
if found is None or self._expired():
|
||||
return None
|
||||
key = (source, expr)
|
||||
if key not in self._bounds:
|
||||
values = await self._coverage.bound_values(found, [expr])
|
||||
self._bounds[key] = values[0] if values else None
|
||||
value = self._bounds[key]
|
||||
if value is None:
|
||||
return None
|
||||
return {"source": source, "expr": expr, "value": str(value), "basis": "live", # type: ignore[typeddict-unknown-key]
|
||||
"at": self._clock()}
|
||||
|
||||
|
||||
def profile_facts(source: Any) -> Callable[[str], Awaitable[Mapping[str, Any] | None]]:
|
||||
"""这个数据源上一张表的剖析事实(catalog_profile_jobs.last_facts 的那一项,同 tools/datasource._coverage 的退回):上传
|
||||
表格的快照对不上这次读的那一版时不用。"""
|
||||
async def facts(table: str) -> Mapping[str, Any] | None:
|
||||
from app.data import catalog as data_catalog
|
||||
from app.data.catalog_profile_jobs import last_facts
|
||||
from app.db.base import SessionLocal
|
||||
|
||||
source_id = getattr(source, "id", None)
|
||||
if not source_id:
|
||||
return None
|
||||
name = data_catalog.resolve_table_name(getattr(source, "schema_cache", None), table) or table
|
||||
async with SessionLocal() as session:
|
||||
found = (await last_facts(session, str(source_id), [name])).get(name)
|
||||
snapshot = getattr(source, "snapshot_id", None)
|
||||
if found and snapshot and found.get("snapshot_id") != snapshot:
|
||||
return None
|
||||
return found
|
||||
|
||||
return facts
|
||||
|
||||
|
||||
def make_prober(sources: Mapping[str, Any], *, masked: Mapping[str, Iterable[str]] | None = None,
|
||||
facts: Mapping[str, Callable[[str], Awaitable[Mapping[str, Any] | None]]] | None = None,
|
||||
cache: Mapping[str, Any] | None = None, budget: float | None = None,
|
||||
clock: Callable[[], str] | None = None) -> LiveProber:
|
||||
"""生成一次分析调用用的探查器(contract.Prober,交进 context["probe"])。
|
||||
|
||||
- sources:数据源名 → 数据源对象(DataSource 行,或上传源绑定快照的视图:同 tools/datasource 建工具时的 source,要有
|
||||
name、kind、options、schema_cache;上传源另有 snapshot_id)。名字和快照里的 source 一致。没给的源不探。
|
||||
- masked:数据源名 → 遮罩列(不分大小写)。缺省按数据源 options 里设的遮罩列(data/engine.masked_columns)。遮罩列上
|
||||
不探,探查器直接交回 None。
|
||||
- facts:数据源名 → 取剖析事实的函数(coverage.Facts:表名 → last_facts 的那一项)。缺省按 profile_facts 现读库(同查询
|
||||
工具的覆盖检查)。
|
||||
- cache:数据源名 → coverage.Cache(同一张表、同一个界一次运行里只查一次);不给就每个探查器各自一份。另外 coverage
|
||||
在进程里把现查到的范围记 5 分钟。
|
||||
- budget:这个探查器所有探查加起来最多多少秒(缺省 coverage.BUDGET_S,从第一次探查算起);过了的一律交回 None。一次
|
||||
分析调用建一个。
|
||||
- clock:现查时刻的写法(测试用);缺省是服务端时区的 ISO 时间(到秒)。
|
||||
"""
|
||||
from app.data import coverage
|
||||
|
||||
return LiveProber(sources, masked=masked, facts=facts, cache=cache,
|
||||
budget=coverage.BUDGET_S if budget is None else budget, clock=clock)
|
||||
|
||||
|
||||
__all__ = ["Book", "LiveProber", "MAX_ROUNDS", "fetch", "make_prober", "probe_key", "profile_facts"]
|
||||
@@ -447,9 +447,14 @@ async def _fallback(flt: _Filter, facts: Facts) -> _Range | None:
|
||||
found = await facts(flt.table_name)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
return _facts_range(found, flt.column_name)
|
||||
|
||||
|
||||
def _facts_range(found: Any, column: str) -> _Range | None:
|
||||
"""剖析事实(last_facts 的一项)里这一列的范围:先剖析的 date_ranges,再取值观察的最新日期。"""
|
||||
if not isinstance(found, Mapping):
|
||||
return None
|
||||
wanted = name_key(flt.column_name)
|
||||
wanted = name_key(column)
|
||||
for item in found.get("date_ranges") or []:
|
||||
if isinstance(item, Mapping) and name_key(str(item.get("column") or "")) == wanted:
|
||||
lo, hi = _as_day(item.get("min")), _as_day(item.get("max"))
|
||||
@@ -806,5 +811,88 @@ def no_data_of(snapshot: Any, column: Any) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
|
||||
__all__ = ["BUDGET_S", "Cache", "NO_DATA_SHORT", "PROBE_TIMEOUT_S", "Window", "filters", "narrowed", "no_data_of",
|
||||
"result_gaps", "sql_gaps", "zero_columns"]
|
||||
# ==========================================================================
|
||||
# 给分析的可比性检查用的几个小函数(app/analysis/comparable.py、probe.py:窗口长度、整期)。读法、写法、探查都和上面同一套,
|
||||
# 只是把中间结果交出去:分析结论要把现查的范围、在库里算出的界记下来,复算时用记下的值
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def parse_sql(sql: Any, read: str | None) -> Any:
|
||||
"""一条查询解析成语法树(同覆盖检查:太长、带模板占位、不止一条、不是查询的为 None)。"""
|
||||
return _parse(sql, read)
|
||||
|
||||
|
||||
def as_day(value: Any, kind: str | None = None) -> _Day | None:
|
||||
"""一个值归一成日期(DATE / DATETIME、'YYYY-MM-DD'、'YYYYMMDD'、8 位整数);认不出为 None。"""
|
||||
return _as_day(value, kind)
|
||||
|
||||
|
||||
def literal_bound(node: Any) -> tuple[bool, _Day | None]:
|
||||
"""时间条件的一个界:(是不是字面量, 字面量归一成的日期)。不是字面量的是按运行当天现算的,要在库里算。"""
|
||||
return (True, _literal_day(node)) if _is_literal(node) else (False, None)
|
||||
|
||||
|
||||
def window_of(conds: list[tuple[str, Any, bool]], days: Mapping[int, _Day | None]) -> Window | None:
|
||||
"""几个条件合成一段区间(同 _window:下界取最晚、上界取最早,「>」「<」的界不带时刻时按天挪一天)。days 是
|
||||
{id(界的节点): 日期}。不核对列和界的写法比不比得起来:分析结论里表的范围只记了日期。"""
|
||||
return _window(conds, days, _Range(None, None, "none"), None)
|
||||
|
||||
|
||||
def _range_out(data: _Range | None) -> dict[str, Any] | None:
|
||||
if data is None:
|
||||
return None
|
||||
return {"min": data.low.day.isoformat() if data.low else None,
|
||||
"max": data.high.day.isoformat() if data.high else None,
|
||||
"basis": data.basis, "at": data.day, "empty": data.empty}
|
||||
|
||||
|
||||
def facts_range(found: Any, column: str) -> dict[str, Any] | None:
|
||||
"""剖析事实(catalog_profile_jobs.last_facts 的一项)里这一列的范围,形状同 column_range;没有为 None。"""
|
||||
return _range_out(_facts_range(found, column))
|
||||
|
||||
|
||||
async def column_range(source: Any, table: str, column: str, *, facts: Facts | None = None,
|
||||
cache: Cache | None = None) -> dict[str, Any] | None:
|
||||
"""这一列在这张表里的实际范围:现查 MIN / MAX(只读守卫、PROBE_TIMEOUT_S 秒、同一张表 LIVE_TTL_S 秒内不重查),查不成
|
||||
退回剖析、取值观察(facts)。{min, max, basis, at, empty}:min / max 是 ISO 日期(不知道的那一头为 None),basis 是 live /
|
||||
profile / observed,at 是剖析、观察的日期(现查为 None),表里这一列一个值都没有时 empty 为真。都没有为 None。从不抛异常。"""
|
||||
from sqlglot import exp
|
||||
|
||||
try:
|
||||
parts = [p for p in str(table).split(".") if p]
|
||||
node = exp.Table(this=exp.to_identifier(parts[-1]),
|
||||
**({"db": exp.to_identifier(parts[-2])} if len(parts) > 1 else {}))
|
||||
flt = _Filter((), node, str(table), exp.column(str(column)), str(column))
|
||||
return _range_out(await _range(source, flt, _read(getattr(source, "kind", None)), cache or Cache(), facts))
|
||||
except Exception: # noqa: BLE001 - 探查失败就当不知道
|
||||
logger.info("可比性检查的范围探查没有成功(数据源 %s)", getattr(source, "name", ""), exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
async def bound_values(source: Any, texts: list[str]) -> list[Any] | None:
|
||||
"""按运行当天现算的几个界(SQL 表达式,按这个数据源的方言写)在库里算出来:一条 SELECT 界1, 界2 …(Oracle 加 FROM
|
||||
DUAL),走只读守卫和 PROBE_TIMEOUT_S。交回各个值(日期、时刻按 ISO 写,其余照库里交回的);查不成为 None。从不抛异常。"""
|
||||
if not texts:
|
||||
return []
|
||||
read = _read(getattr(source, "kind", None))
|
||||
try:
|
||||
found = await _probe(source, _select_sql([f"{t} AS b{i}" for i, t in enumerate(texts)], read))
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
if found is None:
|
||||
return None
|
||||
values = list(found[0]) + [None] * len(texts)
|
||||
out: list[Any] = []
|
||||
for value in values[:len(texts)]:
|
||||
if isinstance(value, datetime):
|
||||
out.append(value.isoformat(sep=" ", timespec="seconds"))
|
||||
elif isinstance(value, date):
|
||||
out.append(value.isoformat())
|
||||
else:
|
||||
out.append(value)
|
||||
return out
|
||||
|
||||
|
||||
__all__ = ["BUDGET_S", "Cache", "NO_DATA_SHORT", "PROBE_TIMEOUT_S", "Window", "as_day", "bound_values", "column_range",
|
||||
"facts_range", "filters", "literal_bound", "narrowed", "no_data_of", "parse_sql", "result_gaps", "sql_gaps",
|
||||
"window_of", "zero_columns"]
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""期次(app/analysis/periods.py):日期、YYYY-MM、YYYY-Www、YYYY-Qn、整数序号认得出、能排序、知道覆盖哪几天;粒度按值推断;
|
||||
写法混着、带时刻的认不出。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from app.analysis import periods as pd
|
||||
|
||||
|
||||
def test_dates_infer_their_grain():
|
||||
items, grain = pd.parse(["2026-10-01", "2026-10-02", "2026-10-04"])
|
||||
assert grain == "day" and [i.start for i in items] == [date(2026, 10, 1), date(2026, 10, 2), date(2026, 10, 4)]
|
||||
items, grain = pd.parse(["2026-09-28", "2026-10-05", "2026-10-19"]) # 隔 7 天的整数倍:周(缺一周)
|
||||
assert grain == "week" and [i.ordinal for i in items] == [0, 1, 3] and items[0].end == date(2026, 10, 4)
|
||||
items, grain = pd.parse(["2026-08-01", "2026-09-01", "2026-10-01"])
|
||||
assert grain == "month" and items[1].end == date(2026, 9, 30)
|
||||
items, grain = pd.parse(["2026-01-01", "2026-04-01", "2026-07-01"])
|
||||
assert grain == "quarter" and items[0].end == date(2026, 3, 31)
|
||||
items, grain = pd.parse([20261001, 20261002])
|
||||
assert grain == "day" and items[1].start == date(2026, 10, 2)
|
||||
assert pd.parse(["2026-10-01 00:00:00", "2026-10-02 00:00:00"])[1] == "day"
|
||||
|
||||
|
||||
def test_labels_for_months_weeks_quarters_and_sequences():
|
||||
items, grain = pd.parse(["2026-09", "2026-10"])
|
||||
assert grain == "month" and items[1].ordinal - items[0].ordinal == 1 and items[1].end == date(2026, 10, 31)
|
||||
items, grain = pd.parse([202609, 202610])
|
||||
assert grain == "month"
|
||||
items, grain = pd.parse(["2026-W41", "2026-W42", "2026-W01"])
|
||||
assert grain == "week" and items[1].start == date(2026, 10, 12) and items[1].end == date(2026, 10, 18)
|
||||
assert items[1].ordinal - items[0].ordinal == 1
|
||||
items, grain = pd.parse(["2025-W52", "2026-W01"]) # 跨年也连着
|
||||
assert items[1].ordinal - items[0].ordinal == 1
|
||||
items, grain = pd.parse(["2026Q3", "2026-Q4"])
|
||||
assert grain == "quarter" and items[1].start == date(2026, 10, 1)
|
||||
items, grain = pd.parse([3, 1, 2])
|
||||
assert grain == "seq" and items[0].start is None and [i.ordinal for i in items] == [3, 1, 2]
|
||||
|
||||
|
||||
def test_unreadable_periods():
|
||||
assert pd.parse(["2026-10-01", "2026-W41"]) is None # 写法混着
|
||||
assert pd.parse(["2026-10-01 08:30:00", "2026-10-02 08:30:00"]) is None # 带时刻
|
||||
assert pd.parse(["上周", "本周"]) is None
|
||||
assert pd.parse(["2026-10", "2026-11"], "week") is None # 粒度和写法对不上
|
||||
assert pd.parse(["2026-10-01", "2026-10-03"], "week") is None # 按周但不隔 7 天
|
||||
assert pd.parse([]) is None and pd.parse([None, "2026-10-01"]) is None
|
||||
|
||||
|
||||
def test_calendar_units_and_weekend_days():
|
||||
assert pd.calendar_unit(date(2026, 9, 1), date(2026, 9, 30)) == "month"
|
||||
assert pd.calendar_unit(date(2026, 7, 1), date(2026, 9, 30)) == "quarter"
|
||||
assert pd.calendar_unit(date(2026, 1, 1), date(2026, 12, 31)) == "year"
|
||||
assert pd.calendar_unit(date(2026, 9, 2), date(2026, 9, 30)) is None
|
||||
assert pd.weekend_days(date(2026, 10, 12), date(2026, 10, 18)) == 2 # 周一到周日
|
||||
assert pd.weekend_days(date(2026, 10, 16), date(2026, 10, 18)) == 2 # 周五到周日
|
||||
assert pd.weekend_days(date(2026, 10, 12), date(2026, 10, 14)) == 0
|
||||
assert pd.weekend_days(date(2026, 10, 1), date(2026, 10, 31)) == 9
|
||||
Reference in New Issue
Block a user