mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 01:11:59 +00:00
合入「口径数字全为 0」的一组修正（部署上一份周报：取到访的那张表只存最新一天，查上周、上上周全是 0，口径卡算出一排 0 和 0%）：查询区间里表里没有数据时照实说——结果里整列是 0 或空时运行时现查（data/coverage.py：界在库里按运行当天算、表的范围现查 MIN / MAX，失败退回剖析，按列判），进查询结果和快照（no_data），口径卡按列标「区间内无数据」、写作目录照实交代、出具按缺口降档、复核记缺口并可按复核重来；助手搭图自查对工具节点写死的 SQL 同样检查、交回一轮修正，单日剖析的表加提示；双盲复算两边落在无数据的列上不判一致、无法复算带上盲算的理由；口径卡分母为 0 时提示写 None、发布检查提醒 else 0、「算不出来」和「缺少输入」分开说；指标清单按格式渲染（以前比率 0.0007 写成 0.0007%）；没配出具契约的成果在实例页印章上照实写缺口。验收：后端全量 14948 通过；方言容器集成测试 PostgreSQL / MySQL / MariaDB / Oracle 全过（含部署那种文字日期 + 现算上周的写法）；tsc、文案检查；相关 8 项浏览器检查 3922 项全过；泄露扫描干净
This commit is contained in:
ThearchyHelios
@@ -207,6 +207,9 @@ NODE_REFERENCE = """\
|
||||
config: {caliber, caliber_version, metrics:[{id, name, unit, decimals, format, expression}], on_missing, assign_to}
|
||||
expression 是受限表达式（不是模板）：vars.kpi.gmv、cell(nodes.fetch, 0, 'gmv')、
|
||||
round((vars.kpi.gmv - vars.kpi.gmv_prev) / vars.kpi.gmv_prev * 100, 1)
|
||||
**分母可能为 0**（上期为 0、这段时间没有数据）时写「… if 分母 else None」：
|
||||
round((vars.kpi.gmv - vars.kpi.gmv_prev) / vars.kpi.gmv_prev * 100, 1) if vars.kpi.gmv_prev else None。
|
||||
算不出来就是空值（报告里显示「—」，写作者照实说算不出来）；不要写 else 0——0 会被读成持平、0%
|
||||
format：thousands（默认，千分位）/ plain（不分组，年份编号这类）/ percent_of_ratio（值是 0.0235 这样的比率，显示成 2.35%）；
|
||||
decimals 写 -15 到 15 的整数；percent_of_ratio 的 decimals 按比率算：要显示两位百分比（2.35%）写 4，
|
||||
写 2 只剩「2%」，0.004 这样的小比率还会显示不出来、报告引用不了。on_missing 默认 fail（缺输入整张卡失败）；写 null 则缺输入的指标记为空值，交给出具契约判档
|
||||
@@ -293,7 +296,9 @@ NODE_REFERENCE = """\
|
||||
6. 问题里的「今天、昨天、上周、本月、去年」这类相对日期一律按运行当天算：SQL 用数据源方言里取当天的写法（CURRENT_DATE、
|
||||
CURDATE()、date('now')、SYSDATE），agent 先调 current_time 拿到当天日期再写条件（agent 的 prompt 里写明这一点、tools 绑上
|
||||
current_time）。不要用表里最大的日期（MAX(日期)）去推「上周」「本月」：数据没更新到那段时间时，推出来的是另一段，报告还会
|
||||
把它当成上周来写。想让报告说清数据最新到哪天，另取一列 MAX(日期) AS 数据最新日期
|
||||
把它当成上周来写。想让报告说清数据最新到哪天，另取一列 MAX(日期) AS 数据最新日期。
|
||||
写按周期取数的 SQL 前先看表的日期范围（look 结果里的剖析、取值观察）：只有一天的数据、或者最新日期早于要查的区间的表，
|
||||
不能用来算这段时间（查出来全是 0）——换有历史的表；没有这样的表，就在说明里照实告诉用户这段时间没有数据
|
||||
|
||||
模型字段（llm / agent / supervisor / report 的 config.model）：
|
||||
- **不要填**。留空表示跟随当前供应商的默认模型，这几乎总是对的。
|
||||
@@ -950,6 +955,7 @@ async def generate(
|
||||
applier.replay_onto(prev_graph)
|
||||
async for _op in gen.repair():
|
||||
pass
|
||||
covered = await gen.checker.coverage(level="warning") if isinstance(gen.checker, GraphChecker) else []
|
||||
|
||||
try:
|
||||
spec = GraphSpec.model_validate({"nodes": list(nodes.values()), "edges": edges})
|
||||
@@ -968,7 +974,7 @@ async def generate(
|
||||
explanation=applier.explanation or str((gen.asked or {}).get("question") or ""),
|
||||
issues=[i.model_dump() for i in report.issues if i.level != "info"]
|
||||
+ _issue_dicts(scope_issues(out["nodes"], scope), "datasource_out_of_scope")
|
||||
+ dropped_tool_warnings(changes, payload.instruction),
|
||||
+ dropped_tool_warnings(changes, payload.instruction) + covered,
|
||||
layout=layout,
|
||||
tool_changes=changes,
|
||||
)
|
||||
@@ -1809,6 +1815,9 @@ async def answer_ops(payload: GenerateIn, *, catalog_view: Any = None):
|
||||
# 能跑，只是跑出来的是模型「假设」查过库的答案
|
||||
changes = tool_changes(baseline, list(nodes.values()))
|
||||
warnings = dropped_tool_warnings(changes, payload.instruction)
|
||||
# 工具节点写死的 SQL 查的区间里表里没有数据（data/coverage.py）：自查时交回模型改过一轮，仍在的不挡运行，留在检查
|
||||
# 结果里当提醒（模型照要求在说明里告诉用户）
|
||||
warnings += await gen.checker.coverage(level="warning") if isinstance(gen.checker, GraphChecker) else []
|
||||
# 建完要不要跑，由模型在 done.run 里表态；问了用户的不跑：要等他回答
|
||||
autorun = applier.autorun and gen.asked is None
|
||||
if remaining:
|
||||
|
||||
@@ -723,6 +723,11 @@ class GraphChecker:
|
||||
self.defaults = defaults
|
||||
self.checkers = checkers
|
||||
self.known = known
|
||||
#: 工具节点写死的 SQL 查的区间里表里没有数据（data/coverage.py）：同一个节点、同一条 SQL 只查一次库
|
||||
self._coverage: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
||||
self._coverage_cache: Any = None
|
||||
#: 这类问题已经交回模型改过一轮：之后不再挡（修正后仍在的留在检查结果里当提醒，由模型在说明里照实告诉用户）
|
||||
self._coverage_given = False
|
||||
|
||||
def blocking(self) -> list[dict[str, Any]] | None:
|
||||
from app.api.copilot import _blocking_issues
|
||||
@@ -735,7 +740,10 @@ class GraphChecker:
|
||||
errors = self.blocking()
|
||||
refused = _refused_issues(self.applier)
|
||||
if self.known is None:
|
||||
return [*refused, *(errors or [])]
|
||||
# 查的区间里表里没有数据：交回模型改一轮（换有这段数据的表，或者在说明里照实告诉用户）；改过一轮仍在的不挡
|
||||
covered = [] if self._coverage_given or errors is None else await self.coverage()
|
||||
self._coverage_given = self._coverage_given or bool(covered)
|
||||
return [*refused, *(errors or []), *covered]
|
||||
from app.engine.autofix import _sig
|
||||
|
||||
if not self.applier.effective:
|
||||
@@ -747,6 +755,77 @@ class GraphChecker:
|
||||
|
||||
return _repair_request(self.applier.nodes, self.applier.edges, issues, looks=looks_text)
|
||||
|
||||
async def coverage(self, *, level: str = "error") -> list[dict[str, Any]]:
|
||||
"""工具节点写死的 SQL 里，查的区间和表里的范围完全不重叠的（data/coverage.sql_gaps）：搭图时没有查询结果，SQL 里
|
||||
所有带时间谓词的表都查（按表去重、限时，不跑原 SQL，只跑算界和现查范围的两条小查询）。一条一个问题（形状同自查的
|
||||
问题，code 是 sql_no_data）；level 给 warning 时是收尾那一步的提醒。"""
|
||||
out: list[dict[str, Any]] = []
|
||||
for nid, node in self.applier.nodes.items():
|
||||
cfg = ((node or {}).get("data") or {}).get("config") or {}
|
||||
tool, args = cfg.get("tool"), cfg.get("args")
|
||||
sql = args.get("sql") if isinstance(args, dict) else None
|
||||
if not isinstance(tool, str) or not tool.startswith("db_query__") or not isinstance(sql, str) \
|
||||
or not sql.strip() or "{{" in sql:
|
||||
continue
|
||||
key = (nid, sql)
|
||||
if key not in self._coverage:
|
||||
self._coverage[key] = await self._gaps(tool[len("db_query__"):], sql)
|
||||
out.extend({"level": level, "node_id": nid, "edge_id": None, "field": "args.sql", "code": "sql_no_data",
|
||||
"message": f"{gap['message']}：这段时间表里没有数据，查出来会是 0",
|
||||
"for_model": f"{gap['message']}：换有这段数据的表；没有这样的表，就在 done 的 explanation 里照实"
|
||||
"告诉用户这段时间表里没有数据、数据只到哪天（写具体日期）"}
|
||||
for gap in self._coverage[key])
|
||||
return out
|
||||
|
||||
async def _gaps(self, name: str, sql: str) -> list[dict[str, Any]]:
|
||||
from app.data import coverage
|
||||
from app.data.engine import masked_columns
|
||||
|
||||
row = next((s for s in self.sources or [] if getattr(s, "name", None) == name), None)
|
||||
if row is None or not getattr(row, "enabled", True):
|
||||
return []
|
||||
if self._coverage_cache is None:
|
||||
self._coverage_cache = coverage.Cache()
|
||||
try:
|
||||
source = await _probe_source(row)
|
||||
except Exception: # noqa: BLE001 - 上传表格没有可用的版本这类：不判
|
||||
return []
|
||||
return await coverage.sql_gaps(source, sql, facts=_profile_facts(row), masked=masked_columns(row.options),
|
||||
cache=self._coverage_cache, budget=COVERAGE_BUDGET_S)
|
||||
|
||||
|
||||
#: 自查时一条 SQL 的覆盖检查最多花多少秒（data/coverage.py）：搭图要等它，比查询时的 BUDGET_S 短
|
||||
COVERAGE_BUDGET_S = 8.0
|
||||
|
||||
|
||||
async def _probe_source(row: Any) -> Any:
|
||||
"""自查时探查用的源：上传表格换成当前版本的快照视图（table_versions.resolve_source），别的原样。"""
|
||||
from app.data import table_versions
|
||||
from app.db.base import SessionLocal
|
||||
|
||||
if (getattr(row, "origin", None) or "manual") != "upload":
|
||||
return row
|
||||
async with SessionLocal() as session:
|
||||
return await table_versions.resolve_source(session, row)
|
||||
|
||||
|
||||
def _profile_facts(row: Any) -> Any:
|
||||
"""现查表的范围不成时退回的剖析事实（catalog_profile_jobs.last_facts）。"""
|
||||
async def facts(table: str) -> Any:
|
||||
from app.data import catalog as data_catalog
|
||||
from app.data.catalog_profile_jobs import last_facts
|
||||
from app.db.base import SessionLocal
|
||||
|
||||
name = data_catalog.resolve_table_name(getattr(row, "schema_cache", None), table) or table
|
||||
async with SessionLocal() as session:
|
||||
found = (await last_facts(session, str(row.id), [name])).get(name)
|
||||
current = getattr(row, "current_snapshot_id", None) if (getattr(row, "origin", None) or "") == "upload" else None
|
||||
if found and current and found.get("snapshot_id") != current:
|
||||
return None # 上传表格：剖析时是另一版数据
|
||||
return found
|
||||
|
||||
return facts
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 应用的落地
|
||||
|
||||
@@ -992,6 +992,10 @@ async def _metric_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[s
|
||||
# 来源查询没通过 SQL 检查：面板标出来并写明原因（问题和改法在查询步骤的 checks 里）。没问题的没有这两个键
|
||||
**({"sql_check_failed": True, "sql_check_reason": source.get("sql_check_reason") or ""}
|
||||
if source.get("sql_check_failed") else {}),
|
||||
# 输入都齐、计算式自己算不出来（分母为 0）：面板说「算不出来」，不说「缺少输入」。有值的没有这个键
|
||||
**({"undefined": True} if source.get("undefined") and source.get("value") is None else {}),
|
||||
# 输入取自「查询区间里这张表没有数据」的列（data/coverage.py）：面板标出来并写明原因。没问题的没有这两个键
|
||||
**({"no_data": True, "no_data_reason": source.get("no_data_reason") or ""} if source.get("no_data") else {}),
|
||||
"node_id": entry.get("node_id"), "artifact": artifact or None, "eid": cite.get("eid"),
|
||||
"eid_ok": cite.get("eid") == entry.get("eid") == make_eid("metric", artifact or None, {"metric": metric_id}),
|
||||
"hash_ok": hash_ok and metric is not None,
|
||||
@@ -1091,6 +1095,8 @@ async def _upstream_steps(metric: dict[str, Any], sealed: _Sealed, masks: "_Mask
|
||||
if found.get("incomplete") else {}),
|
||||
**({"sql_check_failed": True, "sql_check_reason": found.get("sql_check_reason") or ""}
|
||||
if found.get("sql_check_failed") else {}),
|
||||
**({"undefined": True} if found.get("undefined") and found.get("value") is None else {}),
|
||||
**({"no_data": True, "no_data_reason": found.get("no_data_reason") or ""} if found.get("no_data") else {}),
|
||||
"untraced": _untraced_steps(found, sealed, node),
|
||||
})
|
||||
if withheld := {path: _held_word(h) for path, h in held.items() if h.kind != "table"}:
|
||||
@@ -2571,6 +2577,8 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str
|
||||
column_types=snap.get("column_types") if isinstance(snap.get("column_types"), dict) else {},
|
||||
**({"provenance": True} if hint and snap.get("data_version") and not merged else {}),
|
||||
**({"checks": shown.checks(checks)} if (checks := _query_checks(snap)) else {}),
|
||||
# 查询区间里这张表没有数据的那几列（data/coverage.py，查询时记进快照）。没有就不加这个键
|
||||
**({"no_data": gaps} if (gaps := _no_data_view(snap)) else {}),
|
||||
**_catalog_of(snap))
|
||||
if shown.changed:
|
||||
notes.append(SQL_MASKED_NOTE)
|
||||
@@ -2585,6 +2593,17 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str
|
||||
return step
|
||||
|
||||
|
||||
#: 查询步骤上 no_data 每一条给面板的字段（data/coverage.result_gaps 的形状）
|
||||
_NO_DATA_FIELDS = ("columns", "table", "column", "window", "data_range", "basis", "basis_at", "message")
|
||||
|
||||
|
||||
def _no_data_view(snap: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""快照里记的「查询区间里这张表没有数据」（no_data）：只给面板要的那几个字段，认不出的条目不给。"""
|
||||
items = snap.get("no_data") if isinstance(snap.get("no_data"), list) else []
|
||||
return [{k: item[k] for k in _NO_DATA_FIELDS if k in item} for item in items
|
||||
if isinstance(item, dict) and isinstance(item.get("message"), str)]
|
||||
|
||||
|
||||
class _Shown:
|
||||
"""查询步骤上要展示的 SQL、合并 SQL、SQL 检查的片段、合并警告：里面写着的遮罩值（secrets，这次运行查到过的，
|
||||
masking._Secrets）换成「已遮罩」，换了写法的长号码也认。secrets 为 None 时原样交回。changed：换过东西，查询步骤
|
||||
|
||||
@@ -75,6 +75,24 @@ STATUS_TEXT = {"queued": "排队中", "running": "运行中", "interrupted": "
|
||||
REJECTED_TEXT = "被驳回"
|
||||
TIER_TEXT = {"formal": "完整出具", "degraded": "降档出具", "withheld": "不予出具"}
|
||||
NO_TIER = "未声明出具契约"
|
||||
#: 成果没配出具契约（印章上是「未声明出具契约」）时，本该降档的缺口照实写进 notes：不然实例页上看不出来
|
||||
UNGATED_NO_DATA = "指标{names}所依据的查询区间里表里没有数据，这些 0 不代表真的是 0（成果没有配出具契约，没有按缺口降档）"
|
||||
UNGATED_DUAL = "双盲复算有缺口：{gaps}（成果没有配出具契约，没有按缺口降档）"
|
||||
|
||||
|
||||
def ungated_notes(cards: list[dict[str, Any]], dual: dict[str, Any] | None) -> list[str]:
|
||||
"""没配出具契约的成果，印章上要照实写的缺口：口径卡里输入取自「查询区间里这张表没有数据」的列的指标（no_data），
|
||||
双盲复算的缺口（产出的 issuance_gaps：无法复算、审核人没采用原流程……）。配了契约的这些已经按缺口降档，不另写。"""
|
||||
notes: list[str] = []
|
||||
names = list(dict.fromkeys(str(m.get("name") or m.get("id")) for card in cards for m in card.get("metrics") or []
|
||||
if isinstance(m, dict) and m.get("no_data")))
|
||||
if names:
|
||||
shown = "".join(f"「{n}」" for n in names[:3]) + (f"等 {len(names)} 个" if len(names) > 3 else "")
|
||||
notes.append(UNGATED_NO_DATA.format(names=shown))
|
||||
gaps = [str(g) for g in (dual or {}).get("issuance_gaps") or [] if g]
|
||||
if gaps:
|
||||
notes.append(UNGATED_DUAL.format(gaps="；".join(gaps[:2]) + ("等" if len(gaps) > 2 else "")))
|
||||
return notes
|
||||
#: 蓝图试运行（docs/design/normal-user-apps.md N4）：印章的来源、说明，趋势只有一个点时的说明，预览里入口按钮为什么点不了
|
||||
TRIAL_FROM = "试运行"
|
||||
TRIAL_NOTE = "数据是真的，但这不是正式结果：发布之后才会按设定出数"
|
||||
@@ -315,7 +333,8 @@ async def stamp_of(session: AsyncSession, source: Source) -> dict[str, Any]:
|
||||
if run.data_versions:
|
||||
parts.append("数据版本 " + "、".join(_version_text(v) for v in run.data_versions.values() if isinstance(v, dict)))
|
||||
calibers: list[dict[str, str]] = []
|
||||
for _, _, card in _sealed_cards(sealed):
|
||||
cards = [card for _, _, card in _sealed_cards(sealed)]
|
||||
for card in cards:
|
||||
item = {"name": str(card.get("caliber") or ""), "version": str(card.get("caliber_version") or "")}
|
||||
if item["name"] and item not in calibers:
|
||||
calibers.append(item)
|
||||
@@ -339,6 +358,9 @@ async def stamp_of(session: AsyncSession, source: Source) -> dict[str, Any]:
|
||||
# 个人应用最近一次刷新没成功、这里给的是上一次的（_exploratory_source）
|
||||
notes.append(source.message)
|
||||
notes += [gap for entry in (run.data_versions or {}).values() if (gap := _records_gap(entry))]
|
||||
if tier not in TIER_TEXT:
|
||||
# 没配出具契约：双盲复算的缺口、查询区间里没有数据的指标不会降档，照实写在这里
|
||||
notes += ungated_notes(cards, sealed.payload_of(sealed.nodes_of("dual_check")))
|
||||
if not sealed.trusted:
|
||||
notes.append("这次运行还没有封存（或封存核对不一致），数字仅供参考")
|
||||
return {
|
||||
|
||||
@@ -498,9 +498,10 @@ def _snapshot_hidden(content: dict[str, Any], source_masks: Mapping[str, Any]) -
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: 数据源查询工具（tools/datasource.py）落的查询快照里会有的键。有别的键的（比如核查 agent 在查询时就把遮罩列换成
|
||||
#: 「已遮罩」、另记下了遮了哪些列）不知道它记着什么，照现有规则只给 id
|
||||
#: 「已遮罩」、另记下了遮了哪些列）不知道它记着什么，照现有规则只给 id。no_data 是查询区间里这张表没有数据的那几列
|
||||
#: （data/coverage.py）：只有日期和表名、列名，遮罩列上的不探查也不记
|
||||
_QUERY_KEYS = frozenset({"columns", "rows", "row_count", "truncated", "elapsed_ms", "sql", "column_types", "source",
|
||||
"checks", "schema_artifact", "mask_columns", "data_version"})
|
||||
"checks", "schema_artifact", "mask_columns", "data_version", "no_data"})
|
||||
#: 导出时只知道数据源设的遮罩、不知道是哪种库：SQL 按每种支持的库的读法各解析一遍（None 是 sqlglot 的通用读法），
|
||||
#: 至少一种读得通，读得通的每一种都要过关
|
||||
_READINGS = (None, "sqlite", "mysql", "postgres", "oracle")
|
||||
|
||||
@@ -142,11 +142,16 @@ def review_reason(review: Mapping[str, Any]) -> str:
|
||||
|
||||
|
||||
def retry_after_review(review: Mapping[str, Any] | None, attempt: int) -> bool:
|
||||
"""复核之后要不要带着原图重来一次：复核说换个配置重跑大概率就好（retry）、答案不可信（severity=broken）、这一轮还没
|
||||
试满（attempt 从 0 数，attempt + 1 < MAX_ATTEMPTS）。"""
|
||||
"""复核之后要不要带着原图重来一次：复核说换个配置重跑大概率就好（retry）、答案不可信（severity=broken）或者有值得
|
||||
重建一次的缺口（engine/review.REBUILD：查询区间里这张表没有数据，换一张有这段数据的表大概率就好）、这一轮还没试满
|
||||
（attempt 从 0 数，attempt + 1 < MAX_ATTEMPTS）。"""
|
||||
from app.engine.review import REBUILD
|
||||
|
||||
if not review:
|
||||
return False
|
||||
return bool(review.get("retry")) and review.get("severity") == "broken" and attempt + 1 < MAX_ATTEMPTS
|
||||
rebuild = any(isinstance(s, Mapping) and s.get("kind") in REBUILD for s in review.get("signals") or [])
|
||||
return bool(review.get("retry")) and (review.get("severity") == "broken" or rebuild) \
|
||||
and attempt + 1 < MAX_ATTEMPTS
|
||||
|
||||
|
||||
def retry_instruction(question: str, reason: str, has_base: bool) -> str:
|
||||
|
||||
@@ -0,0 +1,810 @@
|
||||
"""查询区间里这张表有没有数据：运行时现查（查询结果整列是 0、助手搭图自查时用）。
|
||||
|
||||
**为什么要有它。** 部署上撞过：一份周报按 `event_date >= DATE_SUB(CURRENT_DATE, INTERVAL (WEEKDAY(CURRENT_DATE)+7) DAY)
|
||||
AND event_date < …` 取上周、上上周的到访人数，那张表却只存最新一天（每天整表重灌），6 列全是 0，口径卡算出一排 0 和 0%，
|
||||
周报照写「上周到访人数为 0 人」「周环比 0.00%」。这些 0 不是「没有到访」，是这段时间表里根本没有数据。剖析记过日期列的
|
||||
最小、最大值，但对滚动表不可靠（剖析那天表里是前一天的数据，运行时已经换成当天），所以要运行时现查。
|
||||
|
||||
**做什么。** 两个入口共用一套：
|
||||
|
||||
- result_gaps：一次查询的结果里有「整列都是 0 或空」的数值列时（结果一行都没有的不归这里管），对每个这样的列，把 SQL
|
||||
收窄到算出这一列的那一项（去掉别的选择项、分组、HAVING、排序、取行数，同双盲复算差异说明的「收窄到这一格」，这里只做
|
||||
分析、不执行），取出它用到的表和时间谓词（列、比较符、界）；
|
||||
- sql_gaps：助手搭图自查时，工具节点里写死的 SQL 还没跑、没有结果：对 SQL 里所有带时间谓词的表都查。
|
||||
|
||||
界是字面量的直接解析；按运行当天现算的（不含列引用的表达式：CURRENT_DATE、DATE_SUB(…)、date('now', …)……）用同一个数据源
|
||||
跑一条 `SELECT 界1, 界2 …` 在库里算出来（Oracle 加 FROM DUAL）。这一列在这张表里的实际范围现查 `SELECT MIN(列), MAX(列)
|
||||
FROM 表`（每条探查 PROBE_TIMEOUT_S 秒）；超时或失败退回剖析、取值观察记下的范围（facts，照实标「按 X 日的剖析」），都没有
|
||||
就不判。日期按常见写法归一（DATE / DATETIME、'YYYY-MM-DD'、'YYYYMMDD'、8 位整数），认不出就不判；列是文本、界也是文本时
|
||||
两边写法要一样（'20261012' 对 '2026-10-12' 按字符串比，结果和按日期比对不上），不一样的不判。区间和实际范围完全不重叠才算
|
||||
「没有数据」；一列用到几段区间（本周减上周）时每一段都不重叠才算。
|
||||
|
||||
**和 sqlcheck 一样宁可漏报。** 所有探查都走数据源查询同一道只读守卫（data/guard.check + is_write），遮罩列上的不探查、不报
|
||||
值；总耗时有上限（BUDGET_S）；解析不了、认不出、探查失败一律当没问题，出错不影响查询本身（同 tools/datasource._sql_checks）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from app.data.names import name_key
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: 一次检查（一份查询结果、一条自查的 SQL）所有探查加起来最多多少秒
|
||||
BUDGET_S = 20.0
|
||||
#: 每条探查查询的时限
|
||||
PROBE_TIMEOUT_S = 10.0
|
||||
#: 一次最多探查几张表的范围、几条按运行当天现算的界：再多就是 SQL 太复杂，不为它把库查一遍
|
||||
MAX_RANGES = 8
|
||||
MAX_BOUNDS = 24
|
||||
#: 太长的 SQL 不分析（同 sqlcheck.MAX_SQL_CHARS）
|
||||
MAX_SQL_CHARS = 20_000
|
||||
#: 一条 no_data 里最多列几个结果列（计数照实）
|
||||
MAX_COLUMNS = 12
|
||||
#: 现查到的表的范围在进程里记多久（秒）：同一张表一会儿内查好几次（一张图里几个取数节点、助手自查几轮）不再全表 MIN /
|
||||
#: MAX。滚动表一天才换一次数据，几分钟的旧不影响判断；查不成的、退回剖析的不记
|
||||
LIVE_TTL_S = 300.0
|
||||
_LIVE: dict[tuple[Any, ...], tuple[float, Any]] = {}
|
||||
#: 现查没成功（大表全表 MIN / MAX 超时、库一时连不上）也记 LIVE_TTL_S：这段时间里同一张表直接退回剖析、取值观察，
|
||||
#: 不然取数 agent 连查几次、每次结果里有整列 0，每次都要白等 PROBE_TIMEOUT_S 秒才退回
|
||||
_PROBE_FAILED = object()
|
||||
|
||||
#: 给人和模型看的说法
|
||||
NO_DATA_MESSAGE = "{columns}查的是 {window}，但 {table} 的 {column} {data}：这些 0 不代表这段时间真的是 0，是这段时间表里没有数据{basis}"
|
||||
NO_DATA_SQL_MESSAGE = "查的区间是 {window}，但 {table} 的 {column} {data}{basis}"
|
||||
BASIS_PROFILE = "（现查表的范围没有成功，按 {day} 的剖析）"
|
||||
BASIS_OBSERVED = "（现查表的范围没有成功，按 {day} 的取值观察）"
|
||||
#: 写进口径卡、写作目录、双盲复算原因的那半句
|
||||
NO_DATA_SHORT = "查询区间里这张表没有数据"
|
||||
|
||||
#: (表达式的类型, 方言) → 按运行当天现算的界：不认列、不认子查询，函数、字面量、INTERVAL 都可以
|
||||
_COMPARE = {"GT": ">", "GTE": ">=", "LT": "<", "LTE": "<=", "EQ": "="}
|
||||
_FLIP = {">": "<", ">=": "<=", "<": ">", "<=": ">=", "=": "="}
|
||||
_ISO = re.compile(r"^(\d{4})-(\d{1,2})-(\d{1,2})(?:[ T](\d{1,2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?)?"
|
||||
r"(?:Z|[+-]\d{2}:?\d{2})?$")
|
||||
_COMPACT = re.compile(r"^(\d{4})(\d{2})(\d{2})$")
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 日期的归一
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Day:
|
||||
"""归一后的一个日期：day 是哪一天，timed 是带不带一天里的时刻（00:00:00 不算），shape 是写法（iso / compact /
|
||||
number / date：库交回的是日期类型），kind 是库交回的类型（date / datetime / text / number，字面量按它的写法）。"""
|
||||
|
||||
day: date
|
||||
timed: bool
|
||||
shape: str
|
||||
kind: str
|
||||
|
||||
|
||||
def _as_day(value: Any, kind: str | None = None) -> _Day | None:
|
||||
"""一个值归一成日期；认不出返回 None。kind 是库交回的列类型（run_query 的 column_types），字面量不给。"""
|
||||
if isinstance(value, bool) or value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return _Day(value.date(), bool(value.time().replace(microsecond=0) != datetime.min.time()), "date", "datetime")
|
||||
if isinstance(value, date):
|
||||
return _Day(value, False, "date", "date")
|
||||
if isinstance(value, (int, float)):
|
||||
if isinstance(value, float) and not value.is_integer():
|
||||
return None
|
||||
found = _COMPACT.match(str(int(value)))
|
||||
day = _ymd(found.group(1), found.group(2), found.group(3)) if found else None
|
||||
return _Day(day, False, "number", kind or "number") if day else None
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
text = value.strip()
|
||||
if found := _COMPACT.match(text):
|
||||
day = _ymd(found.group(1), found.group(2), found.group(3))
|
||||
return _Day(day, False, "compact", kind or "text") if day else None
|
||||
if found := _ISO.match(text):
|
||||
day = _ymd(found.group(1), found.group(2), found.group(3))
|
||||
if day is None:
|
||||
return None
|
||||
timed = any(int(found.group(i) or 0) for i in (4, 5, 6))
|
||||
shape = "date" if kind in ("date", "datetime") else "iso"
|
||||
return _Day(day, timed, shape, kind or "text")
|
||||
return None
|
||||
|
||||
|
||||
def _ymd(y: str, m: str, d: str) -> date | None:
|
||||
try:
|
||||
return date(int(y), int(m), int(d))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _comparable(column: _Day, bound: _Day, *, bare: bool, read: str | None) -> bool:
|
||||
"""库里拿这一列和这个界比，是不是按日期在比（我们的判断才对得上）：
|
||||
|
||||
- 列外面套着函数（DATE(x)、STR_TO_DATE(x, …)）：比的是函数的结果，按日期算；
|
||||
- 列本身是日期类型：库把文本的界转成日期再比；
|
||||
- 界是库算出来的日期类型（DATE_SUB(CURRENT_DATE, …)）、列是文本：MySQL 把文本转成日期再比，别的库不一定（SQLite 的
|
||||
date('now') 交回的是文本，照下一条比写法），只认 MySQL；
|
||||
- 两边都是文本、都是数：写法得一样（'20261012' 对 '2026-10-12' 按字符串比，和按日期比对不上）。"""
|
||||
if not bare or column.kind in ("date", "datetime"):
|
||||
return True
|
||||
if bound.kind in ("date", "datetime"):
|
||||
return read == "mysql" and column.kind == "text"
|
||||
if column.kind == "text" and bound.kind == "text":
|
||||
return column.shape == bound.shape
|
||||
return column.kind == bound.kind == "number" and column.shape == bound.shape == "number"
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 从 SQL 里认时间谓词
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Filter:
|
||||
"""一段 SELECT 里对一张实表的一列的时间条件：table / column 是 sqlglot 的节点（探查时照它写 SQL），conds 是
|
||||
[(比较符, 界的节点, 列是不是光着的)]。key 按内容认（表、列、各个条件的写法）：几列各自收窄出来的同一段条件是同一个，
|
||||
判一次。"""
|
||||
|
||||
key: tuple[Any, ...]
|
||||
table: Any
|
||||
table_name: str
|
||||
column: Any
|
||||
column_name: str
|
||||
conds: list[tuple[str, Any, bool]] = field(default_factory=list)
|
||||
|
||||
|
||||
def _read(kind: str | None) -> str | None:
|
||||
from app.data.sqlcheck import dialect_of
|
||||
|
||||
return dialect_of(kind)
|
||||
|
||||
|
||||
def _parse(sql: str, read: str | None) -> Any:
|
||||
import sqlglot
|
||||
from sqlglot import exp
|
||||
|
||||
if not isinstance(sql, str) or not sql.strip() or len(sql) > MAX_SQL_CHARS or "{{" in sql:
|
||||
return None
|
||||
try:
|
||||
statements = [s for s in sqlglot.parse(sql, read=read) if s is not None]
|
||||
except Exception: # noqa: BLE001 - 解析不了就不判
|
||||
return None
|
||||
if len(statements) != 1 or not isinstance(statements[0], exp.Query):
|
||||
return None
|
||||
return statements[0]
|
||||
|
||||
|
||||
def _output_name(item: Any, read: str | None) -> str:
|
||||
from sqlglot import exp
|
||||
|
||||
if isinstance(item, exp.Alias):
|
||||
return item.alias
|
||||
if isinstance(item, exp.Column):
|
||||
return item.name
|
||||
return item.sql(dialect=read)
|
||||
|
||||
|
||||
def _same_name(a: str, b: str) -> bool:
|
||||
return "".join(a.split()).lower() == "".join(b.split()).lower()
|
||||
|
||||
|
||||
def narrowed(tree: Any, column: str, read: str | None) -> Any:
|
||||
"""收窄到算出 column 这一列的那一项：最外层 SELECT 只留它，分组、HAVING、排序、取行数去掉（只拿来认时间谓词，不执行，
|
||||
不用像双盲复算那样按这一行的分组值补等值条件）。认不准的（不是一段 SELECT、SELECT *、对不上或对上好几项）交回整条。"""
|
||||
from sqlglot import exp
|
||||
|
||||
if not isinstance(tree, exp.Select):
|
||||
return tree
|
||||
items = list(tree.expressions)
|
||||
if any(isinstance(i, exp.Star) or (isinstance(i, exp.Column) and isinstance(i.this, exp.Star)) for i in items):
|
||||
return tree
|
||||
hits = [i for i in items if _same_name(_output_name(i, read), column)]
|
||||
if len(hits) != 1:
|
||||
return tree
|
||||
out = tree.copy()
|
||||
out.set("expressions", [hits[0].copy()])
|
||||
for arg in ("group", "having", "order", "limit", "offset", "qualify"):
|
||||
out.set(arg, None)
|
||||
return out
|
||||
|
||||
|
||||
def _columns_in(node: Any) -> list[Any]:
|
||||
from sqlglot import exp
|
||||
|
||||
return [c for c in node.find_all(exp.Column) if not isinstance(c.this, exp.Star)]
|
||||
|
||||
|
||||
def _plain_bound(node: Any) -> bool:
|
||||
"""界里没有列、子查询、聚合、窗口：字面量，或者按运行当天现算的表达式。"""
|
||||
from sqlglot import exp
|
||||
|
||||
return not _columns_in(node) and not any(
|
||||
isinstance(n, (exp.Select, exp.Subquery, exp.AggFunc, exp.Window, exp.Placeholder, exp.Parameter))
|
||||
for n in node.walk())
|
||||
|
||||
|
||||
def _timeish(subject: Any, column: Any, bounds: list[Any]) -> bool:
|
||||
"""像是时间条件：列名像日期、时间，或者列套着日期函数、界里有日期函数、像日期的常量、按运行当天现算的写法。"""
|
||||
from app.data import sql_semantics as ss
|
||||
|
||||
name = str(column.name or "")
|
||||
return bool(ss._temporal_name(name) or ss._timeish_name(name) or ss._datey(subject)
|
||||
or any(ss._datey(b) or ss._date_like_literal(b) or ss._now_based(b) for b in bounds))
|
||||
|
||||
|
||||
def _condition(cond: Any) -> tuple[Any, Any, list[tuple[str, Any]], bool] | None:
|
||||
"""一个条件是不是「一列 比 界」：(列, 列那一侧的表达式, [(比较符, 界)], 列是不是光着的)。BETWEEN、IN（常量列表）
|
||||
拆成几项；NOT、子查询、两边都有列的不算。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.data.sqlcheck import _bare_column, _unparen
|
||||
|
||||
n = _unparen(cond)
|
||||
if n.args.get("negate"):
|
||||
return None
|
||||
if isinstance(n, exp.Between):
|
||||
subject, pairs = n.this, [(">=", n.args.get("low")), ("<=", n.args.get("high"))]
|
||||
elif isinstance(n, exp.In) and n.expressions and not n.args.get("query"):
|
||||
subject, pairs = n.this, [("in", e) for e in n.expressions]
|
||||
elif type(n).__name__ in _COMPARE and isinstance(n, exp.Binary):
|
||||
op = _COMPARE[type(n).__name__]
|
||||
left, right = n.this, n.expression
|
||||
if _columns_in(left) and not _columns_in(right):
|
||||
subject, pairs = left, [(op, right)]
|
||||
elif _columns_in(right) and not _columns_in(left):
|
||||
subject, pairs = right, [(_FLIP[op], left)]
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
if subject is None or any(b is None or not _plain_bound(b) for _, b in pairs):
|
||||
return None
|
||||
cols = _columns_in(subject)
|
||||
if len(cols) != 1 or any(isinstance(x, (exp.AggFunc, exp.Select, exp.Window)) for x in subject.walk()):
|
||||
return None
|
||||
return cols[0], subject, pairs, _bare_column(subject) is not None
|
||||
|
||||
|
||||
def _real_tables(select: Any, ctes: set[str]) -> dict[str, Any]:
|
||||
"""这一段 FROM / JOIN 里的实表：{别名或表名的 name_key: 表节点}。CTE、子查询不算。"""
|
||||
from sqlglot import exp
|
||||
|
||||
sources = [select.args.get("from_") or select.args.get("from")] + list(select.args.get("joins") or [])
|
||||
out: dict[str, Any] = {}
|
||||
for source in sources:
|
||||
table = getattr(source, "this", None)
|
||||
if isinstance(table, exp.Table) and table.name and name_key(table.name) not in ctes:
|
||||
out[name_key(table.alias_or_name)] = table
|
||||
return out
|
||||
|
||||
|
||||
def _schema_columns(schema: Mapping[str, Any] | None, table: Any) -> set[str] | None:
|
||||
"""表结构里这张表的列（name_key）；表结构里没有这张表为 None。"""
|
||||
tables = (schema or {}).get("tables") if isinstance(schema, Mapping) else None
|
||||
if not isinstance(tables, Mapping):
|
||||
return None
|
||||
for name, meta in tables.items():
|
||||
if name_key(str(name).rsplit(".", 1)[-1]) == name_key(table.name) and isinstance(meta, Mapping):
|
||||
return {name_key(str(c.get("name"))) for c in meta.get("columns") or [] if isinstance(c, Mapping)}
|
||||
return None
|
||||
|
||||
|
||||
def filters(tree: Any, schema: Mapping[str, Any] | None = None) -> list[_Filter]:
|
||||
"""语法树里各段 SELECT 的 WHERE 中，对实表的一列的时间条件，按（段、表、列）归在一起。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.data.sqlcheck import _conjuncts
|
||||
|
||||
ctes = {name_key(c.alias_or_name) for c in tree.find_all(exp.CTE) if c.alias_or_name}
|
||||
found: dict[tuple[int, str, str], _Filter] = {}
|
||||
for index, select in enumerate(tree.find_all(exp.Select)):
|
||||
where = select.args.get("where")
|
||||
if where is None:
|
||||
continue
|
||||
tables = _real_tables(select, ctes)
|
||||
if not tables:
|
||||
continue
|
||||
for cond in _conjuncts(where.this):
|
||||
parsed = _condition(cond)
|
||||
if parsed is None:
|
||||
continue
|
||||
column, subject, pairs, bare = parsed
|
||||
if not _timeish(subject, column, [b for _, b in pairs]):
|
||||
continue
|
||||
if column.table:
|
||||
table = tables.get(name_key(column.table))
|
||||
elif len(tables) == 1:
|
||||
table = next(iter(tables.values()))
|
||||
else:
|
||||
owners = [t for t in tables.values()
|
||||
if name_key(column.name) in (_schema_columns(schema, t) or set())]
|
||||
table = owners[0] if len(owners) == 1 else None
|
||||
if table is None:
|
||||
continue
|
||||
shown = ".".join(p for p in (table.catalog, table.db, table.name) if p)
|
||||
group = (index, name_key(shown), name_key(column.name))
|
||||
entry = found.setdefault(group, _Filter(group, table, shown, column, str(column.name)))
|
||||
entry.conds.extend((op, bound, bare) for op, bound in pairs)
|
||||
for entry in found.values():
|
||||
entry.key = (name_key(entry.table_name), name_key(entry.column_name),
|
||||
tuple(sorted((op, bound.sql(), bare) for op, bound, bare in entry.conds)))
|
||||
return list(found.values())
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 探查
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
async def _probe(source: Any, sql: str) -> tuple[list[Any], list[str | None]] | None:
|
||||
"""在数据源上跑一条探查查询：第一行的值、各列的类型（按位置）。走数据源查询同一道只读守卫；写语句、出错、超时都交回
|
||||
None。"""
|
||||
from app.data.engine import run_query
|
||||
from app.data.guard import QueryLimits, SqlRejected, check, is_write
|
||||
|
||||
kind = (getattr(source, "kind", None) or "").lower()
|
||||
try:
|
||||
statement = check(sql, readonly=True, source_name=getattr(source, "name", ""), dialect=kind)
|
||||
if is_write(statement, dialect=kind):
|
||||
return None
|
||||
result = await run_query(source, statement, limits=QueryLimits(timeout_seconds=PROBE_TIMEOUT_S, max_rows=2))
|
||||
except (SqlRejected, Exception): # noqa: BLE001 - 探查失败就当不知道
|
||||
logger.info("覆盖检查的探查查询没有成功（数据源 %s）", getattr(source, "name", ""), exc_info=True)
|
||||
return None
|
||||
if not result.rows:
|
||||
return None
|
||||
types = result.column_types or {}
|
||||
return list(result.rows[0]), [types.get(c) for c in result.columns]
|
||||
|
||||
|
||||
def _select_sql(items: list[str], read: str | None) -> str:
|
||||
return f"SELECT {', '.join(items)}" + (" FROM DUAL" if read == "oracle" else "")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Cache:
|
||||
"""同一个数据源上探查过的：表里那一列的范围、按运行当天现算的界。助手自查一轮里查好几次，不重复查库。"""
|
||||
|
||||
ranges: dict[tuple[str, str, str], Any] = field(default_factory=dict)
|
||||
bounds: dict[tuple[str, str], Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Range:
|
||||
"""这一列在这张表里的范围。low / high 为 None 的是不知道那一头；empty 是表里这一列一个值都没有（现查出来的）。"""
|
||||
|
||||
low: _Day | None
|
||||
high: _Day | None
|
||||
basis: str
|
||||
day: str | None = None
|
||||
empty: bool = False
|
||||
|
||||
|
||||
#: 剖析事实（catalog_profile_jobs.last_facts 的一项）：表名 → 那一份，取不到为 None
|
||||
Facts = Callable[[str], Awaitable[Mapping[str, Any] | None]]
|
||||
|
||||
|
||||
async def _range(source: Any, flt: _Filter, read: str | None, cache: Cache, facts: Facts | None) -> _Range | None:
|
||||
"""这一列在这张表里的范围：现查 MIN / MAX（同一个数据源、同一张表、同一列查一次），查不成退回剖析、取值观察（facts）。"""
|
||||
from sqlglot import exp
|
||||
|
||||
key = (str(getattr(source, "name", "")), name_key(flt.table_name), name_key(flt.column_name))
|
||||
if key in cache.ranges:
|
||||
return cache.ranges[key]
|
||||
live = (getattr(source, "id", None), getattr(source, "snapshot_id", None), *key)
|
||||
held = _LIVE.get(live)
|
||||
failed_before = False
|
||||
if held is not None and held[0] > time.monotonic():
|
||||
if held[1] is not _PROBE_FAILED:
|
||||
cache.ranges[key] = held[1]
|
||||
return held[1]
|
||||
failed_before = True
|
||||
table = flt.table.copy()
|
||||
table.set("alias", None)
|
||||
ident = flt.column.this.copy() if isinstance(flt.column.this, exp.Identifier) \
|
||||
else exp.to_identifier(flt.column_name)
|
||||
column = exp.Column(this=ident)
|
||||
sql = (f"SELECT {exp.Min(this=column.copy()).sql(dialect=read)} AS lo, "
|
||||
f"{exp.Max(this=column.copy()).sql(dialect=read)} AS hi FROM {table.sql(dialect=read)}")
|
||||
found = None if failed_before else await _probe(source, sql)
|
||||
out: _Range | None = None
|
||||
if found is not None:
|
||||
values, kinds = found
|
||||
low_raw, high_raw = (values + [None, None])[:2]
|
||||
kinds = (kinds + [None, None])[:2]
|
||||
if low_raw is None and high_raw is None:
|
||||
out = _Range(None, None, "live", empty=True)
|
||||
else:
|
||||
low, high = _as_day(low_raw, kinds[0]), _as_day(high_raw, kinds[1])
|
||||
out = _Range(low, high, "live") if low is not None and high is not None else None
|
||||
elif facts is not None:
|
||||
out = await _fallback(flt, facts)
|
||||
cache.ranges[key] = out
|
||||
if found is None and not failed_before:
|
||||
_remember(live, _PROBE_FAILED)
|
||||
elif out is not None and out.basis == "live":
|
||||
_remember(live, out)
|
||||
return out
|
||||
|
||||
|
||||
def _remember(live: tuple[Any, ...], value: Any) -> None:
|
||||
if len(_LIVE) > 512:
|
||||
_LIVE.clear()
|
||||
_LIVE[live] = (time.monotonic() + LIVE_TTL_S, value)
|
||||
|
||||
|
||||
async def _fallback(flt: _Filter, facts: Facts) -> _Range | None:
|
||||
"""现查不成：退回剖析记下的范围（date_ranges），再不然取值观察的最新日期（freshness，只知道最晚那一头）。"""
|
||||
try:
|
||||
found = await facts(flt.table_name)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
if not isinstance(found, Mapping):
|
||||
return None
|
||||
wanted = name_key(flt.column_name)
|
||||
for item in found.get("date_ranges") or []:
|
||||
if isinstance(item, Mapping) and name_key(str(item.get("column") or "")) == wanted:
|
||||
lo, hi = _as_day(item.get("min")), _as_day(item.get("max"))
|
||||
if lo is not None and hi is not None:
|
||||
return _Range(lo, hi, "profile", day=str(found.get("at") or "")[:10] or None)
|
||||
observed = found.get("observed") if isinstance(found.get("observed"), Mapping) else {}
|
||||
for item in observed.get("freshness") or []:
|
||||
if isinstance(item, Mapping) and name_key(str(item.get("column") or "")) == wanted:
|
||||
hi = _as_day(item.get("max"))
|
||||
if hi is not None:
|
||||
return _Range(None, hi, "observed", day=str(item.get("at") or "")[:10] or None)
|
||||
return None
|
||||
|
||||
|
||||
def _literal_day(node: Any) -> _Day | None:
|
||||
from sqlglot import exp
|
||||
|
||||
from app.data.sqlcheck import _unparen
|
||||
|
||||
node = _unparen(node)
|
||||
negative = False
|
||||
if isinstance(node, exp.Neg):
|
||||
node, negative = _unparen(node.this), True
|
||||
if not isinstance(node, exp.Literal) or negative:
|
||||
return None
|
||||
if node.is_string:
|
||||
return _as_day(str(node.this))
|
||||
try:
|
||||
value = int(str(node.this))
|
||||
except ValueError:
|
||||
return None
|
||||
return _as_day(value)
|
||||
|
||||
|
||||
def _is_literal(node: Any) -> bool:
|
||||
from sqlglot import exp
|
||||
|
||||
from app.data.sqlcheck import _unparen
|
||||
|
||||
node = _unparen(node)
|
||||
if isinstance(node, exp.Neg):
|
||||
node = _unparen(node.this)
|
||||
return isinstance(node, exp.Literal)
|
||||
|
||||
|
||||
async def _bounds(source: Any, nodes: list[Any], read: str | None, cache: Cache) -> dict[int, _Day | None]:
|
||||
"""各个界归一成日期：字面量直接解析，别的（按运行当天现算的表达式）一条 SELECT 在库里算出来。{id(节点): 日期}。"""
|
||||
out: dict[int, _Day | None] = {}
|
||||
pending: dict[str, list[int]] = {}
|
||||
name = str(getattr(source, "name", ""))
|
||||
for node in nodes:
|
||||
if _is_literal(node):
|
||||
out[id(node)] = _literal_day(node)
|
||||
continue
|
||||
try:
|
||||
text = node.sql(dialect=read)
|
||||
except Exception: # noqa: BLE001
|
||||
out[id(node)] = None
|
||||
continue
|
||||
if (name, text) in cache.bounds:
|
||||
out[id(node)] = cache.bounds[(name, text)]
|
||||
else:
|
||||
pending.setdefault(text, []).append(id(node))
|
||||
texts = list(pending)[:MAX_BOUNDS]
|
||||
found = await _probe(source, _select_sql([f"{t} AS b{i}" for i, t in enumerate(texts)], read)) if texts else None
|
||||
for i, text in enumerate(pending):
|
||||
day = None
|
||||
if found is not None and i < len(texts):
|
||||
values, kinds = found
|
||||
day = _as_day(values[i], kinds[i]) if i < len(values) and i < len(kinds) else None
|
||||
if i < len(texts):
|
||||
cache.bounds[(name, text)] = day
|
||||
for ident in pending[text]:
|
||||
out[ident] = day
|
||||
return out
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Window:
|
||||
"""一段时间条件的范围（按天，两头都含）。start / end 为 None 的是那一头没有限制。"""
|
||||
|
||||
start: date | None
|
||||
end: date | None
|
||||
|
||||
|
||||
def _window(conds: list[tuple[str, Any, bool]], days: Mapping[int, _Day | None], data: _Range,
|
||||
read: str | None) -> Window | None:
|
||||
"""几个条件合成一段：下界取最晚的、上界取最早的；IN 列表取最早到最晚（中间没列到的日子也算进去，宁可判成有重叠）。
|
||||
有一个界认不出、或者和列比不起来，整段不判（None）。「>」「<」的界不带时刻时按天挪一天（> 2026-10-11 从 10-12 算起）；
|
||||
带时刻的不挪（那一天里还可能有数据，宁可判成有重叠）。"""
|
||||
lows: list[date] = []
|
||||
highs: list[date] = []
|
||||
points: list[date] = []
|
||||
sample = data.high or data.low
|
||||
for op, node, bare in conds:
|
||||
day = days.get(id(node))
|
||||
if day is None or (sample is not None and not _comparable(sample, day, bare=bare, read=read)):
|
||||
return None
|
||||
if op == "in":
|
||||
points.append(day.day)
|
||||
continue
|
||||
if op in (">=", "="):
|
||||
lows.append(day.day)
|
||||
elif op == ">":
|
||||
lows.append(day.day if day.timed else day.day + timedelta(days=1))
|
||||
if op in ("<=", "="):
|
||||
highs.append(day.day)
|
||||
elif op == "<":
|
||||
highs.append(day.day if day.timed else day.day - timedelta(days=1))
|
||||
if points:
|
||||
lows.append(min(points))
|
||||
highs.append(max(points))
|
||||
start = max(lows) if lows else None
|
||||
end = min(highs) if highs else None
|
||||
if start is None and end is None:
|
||||
return None
|
||||
if start is not None and end is not None and start > end:
|
||||
return None
|
||||
return Window(start, end)
|
||||
|
||||
|
||||
def _disjoint(window: Window, data: _Range) -> bool:
|
||||
"""区间和表里的范围完全不重叠（表里这一列一个值都没有也算）。"""
|
||||
if data.empty:
|
||||
return True
|
||||
if window.end is not None and data.low is not None and window.end < data.low.day:
|
||||
return True
|
||||
return window.start is not None and data.high is not None and window.start > data.high.day
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 说法
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def _window_text(window: Window) -> str:
|
||||
if window.start is not None and window.end is not None:
|
||||
return window.start.isoformat() if window.start == window.end \
|
||||
else f"{window.start.isoformat()} 到 {window.end.isoformat()}"
|
||||
if window.start is not None:
|
||||
return f"{window.start.isoformat()} 起"
|
||||
return f"{window.end.isoformat()} 及以前" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def _data_text(data: _Range) -> str:
|
||||
if data.empty:
|
||||
return "一个值都没有"
|
||||
if data.low is not None and data.high is not None:
|
||||
if data.low.day == data.high.day:
|
||||
return f"只有 {data.low.day.isoformat()} 这一天的数据"
|
||||
return f"只有 {data.low.day.isoformat()} 到 {data.high.day.isoformat()} 的数据"
|
||||
return f"最新只到 {data.high.day.isoformat()}" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def _basis_text(data: _Range) -> str:
|
||||
if data.basis == "profile":
|
||||
return BASIS_PROFILE.format(day=data.day or "上次")
|
||||
if data.basis == "observed":
|
||||
return BASIS_OBSERVED.format(day=data.day or "上次")
|
||||
return ""
|
||||
|
||||
|
||||
def _columns_text(columns: list[str]) -> str:
|
||||
head = f"「{columns[0]}」"
|
||||
return f"{head}等 {len(columns)} 列" if len(columns) > 1 else head
|
||||
|
||||
|
||||
def _range_dict(data: _Range) -> list[str | None] | None:
|
||||
if data.empty:
|
||||
return None
|
||||
return [data.low.day.isoformat() if data.low else None, data.high.day.isoformat() if data.high else None]
|
||||
|
||||
|
||||
def _window_dict(window: Window) -> list[str | None]:
|
||||
return [window.start.isoformat() if window.start else None, window.end.isoformat() if window.end else None]
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 入口
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def zero_columns(payload: Mapping[str, Any]) -> list[str]:
|
||||
"""查询结果里「整列都是 0 或空」的数值列（按 column_types 是数的、或者没记类型却全是数和空值的）。一行都没有的结果
|
||||
返回空列表：那归「没有查到」管，不归这里。"""
|
||||
from app.engine.expressions import numeric_text
|
||||
|
||||
columns = [str(c) for c in payload.get("columns") or []]
|
||||
rows = payload.get("rows") or []
|
||||
if not columns or not isinstance(rows, list) or not rows:
|
||||
return []
|
||||
types = payload.get("column_types") if isinstance(payload.get("column_types"), Mapping) else {}
|
||||
out: list[str] = []
|
||||
for index, column in enumerate(columns):
|
||||
if types.get(column) not in (None, "number"):
|
||||
continue
|
||||
values = [row[index] if isinstance(row, (list, tuple)) and index < len(row)
|
||||
else (row.get(column) if isinstance(row, Mapping) else None) for row in rows]
|
||||
|
||||
def zero(value: Any) -> bool:
|
||||
if value is None:
|
||||
return True
|
||||
if isinstance(value, bool):
|
||||
return False
|
||||
if isinstance(value, (int, float)):
|
||||
return value == 0
|
||||
number = numeric_text(value) if isinstance(value, str) else None
|
||||
return number is not None and number == 0
|
||||
|
||||
if all(zero(v) for v in values):
|
||||
out.append(column)
|
||||
return out
|
||||
|
||||
|
||||
async def _judge(source: Any, flts: list[_Filter], read: str | None, cache: Cache, facts: Facts | None,
|
||||
masked: set[str], deadline: float) -> dict[tuple[int, str, str], tuple[str, Window | None, _Range | None]]:
|
||||
"""每段时间条件判一下：none（区间里表里没有数据）/ some（有重叠）/ unknown（判断不了），带上区间和范围。"""
|
||||
out: dict[tuple[int, str, str], tuple[str, Window | None, _Range | None]] = {}
|
||||
ranges: dict[tuple[str, str], _Range | None] = {}
|
||||
usable = [f for f in flts if name_key(f.column_name) not in masked]
|
||||
for flt in flts:
|
||||
if flt not in usable:
|
||||
out[flt.key] = ("unknown", None, None)
|
||||
for flt in usable:
|
||||
if time.monotonic() > deadline:
|
||||
out[flt.key] = ("unknown", None, None)
|
||||
continue
|
||||
rkey = (name_key(flt.table_name), name_key(flt.column_name))
|
||||
if rkey not in ranges:
|
||||
ranges[rkey] = await _range(source, flt, read, cache, facts) if len(ranges) < MAX_RANGES else None
|
||||
data = ranges[rkey]
|
||||
if data is None:
|
||||
out[flt.key] = ("unknown", None, None)
|
||||
continue
|
||||
days = await _bounds(source, [node for _, node, _ in flt.conds], read, cache)
|
||||
window = _window(flt.conds, days, data, read)
|
||||
if window is None:
|
||||
out[flt.key] = ("unknown", None, data)
|
||||
else:
|
||||
out[flt.key] = ("none" if _disjoint(window, data) else "some", window, data)
|
||||
return out
|
||||
|
||||
|
||||
async def result_gaps(source: Any, sql: str, payload: Mapping[str, Any], *, facts: Facts | None = None,
|
||||
masked: Iterable[str] = (), cache: Cache | None = None,
|
||||
budget: float = BUDGET_S) -> list[dict[str, Any]]:
|
||||
"""一份查询结果里，整列是 0 或空、而查询区间里这张表没有数据的那几列：[{columns, table, column, window, data_range,
|
||||
basis, basis_at?, message}]，同一张表、同一列、同一段区间的几列并成一条。没有问题（或者判断不了）返回空列表。从不抛异常。"""
|
||||
try:
|
||||
return await asyncio.wait_for(_result_gaps(source, sql, payload, facts, set(masked), cache or Cache(), budget),
|
||||
timeout=budget + PROBE_TIMEOUT_S)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("覆盖检查超过时限，本次跳过（数据源 %s）", getattr(source, "name", ""))
|
||||
except Exception: # noqa: BLE001 - 检查不能让一次成功的查询变成失败
|
||||
logger.exception("覆盖检查出错，本次跳过（数据源 %s）", getattr(source, "name", ""))
|
||||
return []
|
||||
|
||||
|
||||
async def _result_gaps(source: Any, sql: str, payload: Mapping[str, Any], facts: Facts | None, masked: set[str],
|
||||
cache: Cache, budget: float) -> list[dict[str, Any]]:
|
||||
zeros = zero_columns(payload)
|
||||
if not zeros:
|
||||
return []
|
||||
read = _read(getattr(source, "kind", None))
|
||||
tree = _parse(sql, read)
|
||||
if tree is None:
|
||||
return []
|
||||
masked = {m.strip().lower() for m in masked if isinstance(m, str) and m.strip()}
|
||||
schema = getattr(source, "schema_cache", None)
|
||||
per_column = {c: filters(narrowed(tree, c, read), schema) for c in zeros if c.lower() not in masked}
|
||||
every = {f.key: f for flts in per_column.values() for f in flts}
|
||||
if not every:
|
||||
return []
|
||||
judged = await _judge(source, list(every.values()), read, cache, facts, masked, time.monotonic() + budget)
|
||||
groups: dict[tuple[Any, ...], dict[str, Any]] = {}
|
||||
for column, flts in per_column.items():
|
||||
verdicts = [judged.get(f.key) for f in flts]
|
||||
if not flts or any(v is None or v[0] != "none" for v in verdicts):
|
||||
continue
|
||||
for flt, (_, window, data) in zip(flts, verdicts):
|
||||
gkey = (name_key(flt.table_name), name_key(flt.column_name), window, data.basis if data else None)
|
||||
group = groups.setdefault(gkey, {"flt": flt, "window": window, "data": data, "columns": []})
|
||||
if column not in group["columns"]:
|
||||
group["columns"].append(column)
|
||||
out = []
|
||||
for group in groups.values():
|
||||
flt, window, data, columns = group["flt"], group["window"], group["data"], group["columns"]
|
||||
out.append({
|
||||
"columns": columns[:MAX_COLUMNS], "table": flt.table_name, "column": flt.column_name,
|
||||
"window": _window_dict(window), "data_range": _range_dict(data), "basis": data.basis,
|
||||
**({"basis_at": data.day} if data.day else {}),
|
||||
"message": NO_DATA_MESSAGE.format(columns=_columns_text(columns), window=_window_text(window),
|
||||
table=flt.table_name, column=flt.column_name,
|
||||
data=_data_text(data), basis=_basis_text(data)),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
async def sql_gaps(source: Any, sql: str, *, facts: Facts | None = None, masked: Iterable[str] = (),
|
||||
cache: Cache | None = None, budget: float = BUDGET_S) -> list[dict[str, Any]]:
|
||||
"""还没执行的一条 SQL（助手自查）：SQL 里每一段时间条件，区间和表里的范围完全不重叠的那几段：[{table, column,
|
||||
window, data_range, basis, basis_at?, message}]，同一张表、同一列、同一段区间只算一条。从不抛异常。"""
|
||||
try:
|
||||
return await asyncio.wait_for(_sql_gaps(source, sql, facts, set(masked), cache or Cache(), budget),
|
||||
timeout=budget + PROBE_TIMEOUT_S)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("自查的覆盖检查超过时限，本次跳过（数据源 %s）", getattr(source, "name", ""))
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("自查的覆盖检查出错，本次跳过（数据源 %s）", getattr(source, "name", ""))
|
||||
return []
|
||||
|
||||
|
||||
async def _sql_gaps(source: Any, sql: str, facts: Facts | None, masked: set[str], cache: Cache,
|
||||
budget: float) -> list[dict[str, Any]]:
|
||||
read = _read(getattr(source, "kind", None))
|
||||
tree = _parse(sql, read)
|
||||
if tree is None:
|
||||
return []
|
||||
masked = {m.strip().lower() for m in masked if isinstance(m, str) and m.strip()}
|
||||
flts = filters(tree, getattr(source, "schema_cache", None))
|
||||
if not flts:
|
||||
return []
|
||||
judged = await _judge(source, flts, read, cache, facts, masked, time.monotonic() + budget)
|
||||
out: list[dict[str, Any]] = []
|
||||
seen: set[tuple[Any, ...]] = set()
|
||||
for flt in flts:
|
||||
verdict, window, data = judged.get(flt.key, ("unknown", None, None))
|
||||
key = (name_key(flt.table_name), name_key(flt.column_name), window)
|
||||
if verdict != "none" or window is None or data is None or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append({"table": flt.table_name, "column": flt.column_name, "window": _window_dict(window),
|
||||
"data_range": _range_dict(data), "basis": data.basis, **({"basis_at": data.day} if data.day else {}),
|
||||
"message": NO_DATA_SQL_MESSAGE.format(window=_window_text(window), table=flt.table_name,
|
||||
column=flt.column_name, data=_data_text(data),
|
||||
basis=_basis_text(data))})
|
||||
return out
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 读快照里记下的
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def no_data_of(snapshot: Any, column: Any) -> dict[str, Any] | None:
|
||||
"""查询快照里记着 column 这一列「查询区间里这张表没有数据」的那一条（no_data）；没有为 None。按列名认（不分大小写、
|
||||
不计空白）。"""
|
||||
if not isinstance(snapshot, Mapping) or not isinstance(column, str):
|
||||
return None
|
||||
for item in snapshot.get("no_data") or []:
|
||||
if isinstance(item, Mapping) and any(isinstance(c, str) and _same_name(c, column)
|
||||
for c in item.get("columns") or []):
|
||||
return dict(item)
|
||||
return None
|
||||
|
||||
|
||||
__all__ = ["BUDGET_S", "Cache", "NO_DATA_SHORT", "PROBE_TIMEOUT_S", "Window", "filters", "narrowed", "no_data_of",
|
||||
"result_gaps", "sql_gaps", "zero_columns"]
|
||||
@@ -69,6 +69,8 @@ PROFILE_ROWS_PROMPT = {"stats": "约 {n} 行（数据库统计估算）", "count
|
||||
"at_least": "至少 {n} 行（数到上限为止）"}
|
||||
PROFILE_RANGE_PROMPT = "{column} 从 {low} 到 {high}"
|
||||
PROFILE_HINT_PROMPT = "只供写查询参考（定时间范围、估表大小），不是查询结果，不能作为回答里的数"
|
||||
#: 日期范围最早和最晚是同一天：这张表很可能只存最新一天（每天整表重灌），拿它算上周、上月只会查出 0
|
||||
PROFILE_ONE_DAY_PROMPT = "剖析时这张表的 {column} 只有一天（{day}）的数据，可能每天整表重灌：算历史周期请换有历史的表"
|
||||
PROFILE_STALE_PROMPT = "数据剖析：剖析早于当前版本，不给"
|
||||
#: 取值观察的结论（next-round 03 B2）：最新业务日期；行数比剖析的新时用观察的
|
||||
PROFILE_LATEST_PROMPT = "最新业务日期 {column} 到 {value}（取值观察，{day}）"
|
||||
@@ -241,6 +243,7 @@ def profile_lines(facts: Mapping[str, Any] | None, masked: set[str], snapshot_id
|
||||
if upload and (not snapshot_id or facts.get("snapshot_id") != snapshot_id):
|
||||
return [indent + PROFILE_STALE_PROMPT]
|
||||
parts: list[str] = []
|
||||
one_day: list[str] = []
|
||||
estimate = facts.get("row_estimate") if isinstance(facts.get("row_estimate"), Mapping) else None
|
||||
seen_rows = observed.get("rows") if isinstance(observed.get("rows"), Mapping) else None
|
||||
if seen_rows and str(seen_rows.get("at") or "") > str(facts.get("at") or ""):
|
||||
@@ -259,6 +262,8 @@ def profile_lines(facts: Mapping[str, Any] | None, masked: set[str], snapshot_id
|
||||
if not column or column.lower() in masked or found.get("min") is None or found.get("max") is None:
|
||||
continue
|
||||
parts.append(PROFILE_RANGE_PROMPT.format(column=column, low=_when(found["min"]), high=_when(found["max"])))
|
||||
if (day := _same_day(found["min"], found["max"])) is not None:
|
||||
one_day.append(PROFILE_ONE_DAY_PROMPT.format(column=column, day=day))
|
||||
for found in observed.get("freshness") or []:
|
||||
column = str((found or {}).get("column") or "")
|
||||
if column and column.lower() not in masked and found.get("max"):
|
||||
@@ -271,7 +276,15 @@ def profile_lines(facts: Mapping[str, Any] | None, masked: set[str], snapshot_id
|
||||
# 只有取值观察、没剖析过：照实说是观察
|
||||
latest = max((str(f.get("at") or "") for f in [seen_rows or {}, *(observed.get("freshness") or [])]), default="")
|
||||
head = OBSERVE_HEAD_PROMPT.format(day=_day(latest), facts="；".join(parts))
|
||||
return [indent + head, indent + " " + PROFILE_HINT_PROMPT]
|
||||
return [indent + head, *(indent + " " + line for line in one_day), indent + " " + PROFILE_HINT_PROMPT]
|
||||
|
||||
|
||||
def _same_day(low: Any, high: Any) -> str | None:
|
||||
"""日期范围的两头是同一天（DATE、DATETIME、'YYYY-MM-DD'、'YYYYMMDD' 都认，data/coverage 同一个归一）：那一天，不是为 None。"""
|
||||
from app.data.coverage import _as_day
|
||||
|
||||
lo, hi = _as_day(low), _as_day(high)
|
||||
return lo.day.isoformat() if lo is not None and hi is not None and lo.day == hi.day else None
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
|
||||
@@ -201,6 +201,16 @@ class Side:
|
||||
inputs: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def missing_reason(side: str, metric: dict[str, Any]) -> str | None:
|
||||
"""一边的指标没有值时的那句原因：输入都齐、计算式自己算不出来的（口径卡记了 undefined，分母为 0）说「算不出来」，
|
||||
别的说「缺少输入」。有值的为 None。"""
|
||||
if metric.get("status") != "missing_input":
|
||||
return None
|
||||
if metric.get("undefined"):
|
||||
return f"{side}这个指标算不出来（这组输入下计算式没有值，比如分母为 0）"
|
||||
return f"{side}这个指标缺少输入，没有值"
|
||||
|
||||
|
||||
def compare(base: Side, target: Side) -> dict[str, Any]:
|
||||
"""对比两次运行（base 是基准，target 是这次），只读，不碰两次运行的任何东西。返回：
|
||||
|
||||
@@ -1499,8 +1509,8 @@ class _Compare:
|
||||
out.update(classification=DATA_ONLY, reasons=[
|
||||
DATA_FROM_UPSTREAM if self.template["same_graph"] else DATA_FROM_UPSTREAM_SCOPED])
|
||||
for side, metric in (("基准运行", bm), ("这次运行", tm)):
|
||||
if metric.get("status") == "missing_input":
|
||||
out["reasons"].append(f"{side}这个指标缺少输入，没有值")
|
||||
if said := missing_reason(side, metric):
|
||||
out["reasons"].append(said)
|
||||
if out["classification"] in _ATTRIBUTED and bp is not None and tp is not None:
|
||||
out["attribution"] = self._attribution(bp, tp, card_id)
|
||||
if any(inp.masked for plan in (bp, tp) if plan is not None for inp in plan.inputs.values()):
|
||||
|
||||
@@ -926,6 +926,12 @@ def build_catalog(
|
||||
# 来源查询没通过 SQL 检查（data/sqlcheck.py 的 error 级问题）：同样只在有问题时有这两个键
|
||||
**({"sql_check_failed": True, "sql_check_reason": metric.get("sql_check_reason") or ""}
|
||||
if metric.get("sql_check_failed") else {}),
|
||||
# 输入都齐、计算式自己算不出来（分母为 0）：写作目录、引用解析不了的原因说「算不出来」，不说「缺少输入」
|
||||
**({"undefined": True} if metric.get("undefined") and metric.get("value") is None else {}),
|
||||
# 输入取自「查询区间里这张表没有数据」的列（data/coverage.py）：写作目录里提醒写作者，证据面板标出来。
|
||||
# 只在有问题时有这两个键
|
||||
**({"no_data": True, "no_data_reason": metric.get("no_data_reason") or ""}
|
||||
if metric.get("no_data") else {}),
|
||||
}
|
||||
|
||||
queries = retrievals = 0
|
||||
@@ -1653,7 +1659,8 @@ def resolve_marker(marker: dict[str, Any], catalog: dict[str, Any], *, cells_all
|
||||
# 换算了只会解析不了、原因说成「不是数值」。引用它本身由报告自查另记一条违规（masked_metrics）
|
||||
rendered = str(entry.get("rendered") or "")
|
||||
elif entry.get("value") is None:
|
||||
return _unresolved(marker, alias, ev_kind, f"指标「{entry.get('name') or ref}」本次没有值（缺少输入），"
|
||||
why = UNDEFINED_REASON if entry.get("undefined") else "缺少输入"
|
||||
return _unresolved(marker, alias, ev_kind, f"指标「{entry.get('name') or ref}」本次没有值（{why}），"
|
||||
"无法写入报告", rendered=MISSING)
|
||||
else:
|
||||
try:
|
||||
@@ -3450,6 +3457,17 @@ SEE_AND_CODE_RULES = (
|
||||
"（「状态为 9」「类型 1」）：码值是没有出处的数字，系统会当成裸数字打回重写。"
|
||||
)
|
||||
|
||||
#: 查询区间里这张表没有数据（查询快照的 no_data、口径卡的 no_data，data/coverage.py）：写作目录里的两种提醒、写作规则。
|
||||
#: 都带着 NO_DATA_MARK 这几个字：报告节点据此决定要不要加 NO_DATA_RULE
|
||||
NO_DATA_MARK = "这段时间表里没有数据"
|
||||
NO_DATA_METRIC_NOTE = "（这段时间表里没有数据：所依据的查询区间在表里查不到数据，这个数不要当成 0 解读，照实写没有数据）"
|
||||
NO_DATA_QUERY_NOTE = " （注意：{message}。引用这几列时照实写这段时间表里没有数据，不要当成 0 解读）"
|
||||
NO_DATA_RULE = ("标着「这段时间表里没有数据」的指标和查询列：不要写成 0、持平或 0%，照实写这段时间表里没有数据、"
|
||||
"数据只到哪天（写具体日期）。")
|
||||
|
||||
#: 指标「算不出来」的那句（口径卡记了 undefined：输入都齐、计算式自己交回空值）：引用解析不了的原因、写作目录里用
|
||||
UNDEFINED_REASON = "算不出来：这组输入下计算式没有值，比如分母为 0"
|
||||
|
||||
#: [[m:]]、[[v:]] 渲染出来自带单位（部署上撞过「覆盖 8条 条产线」：口径卡的单位是「条」，写作者照着助手的指令又写了一个）。
|
||||
#: 渲染时紧跟着的同一个单位会去掉（_tidy），这里先说给写作者听
|
||||
UNIT_RULE = ("数字标记渲染出来自带单位（[[m:…]] 是口径卡里的单位，比如「8条」「12,345人次」），后面不要再写一遍单位："
|
||||
@@ -3725,6 +3743,10 @@ def _query_prompt(entry: dict[str, Any], snaps: _Snapshots, cells_allowed: bool,
|
||||
"不要把取回的行数当成总数，也不要据此写合计、全部之类的结论）")
|
||||
if note := _sql_check_note(entry.get("artifact"), snaps):
|
||||
lines.append(note)
|
||||
for gap in snapshot.get("no_data") or [] if isinstance(snapshot.get("no_data"), list) else []:
|
||||
# 查询区间里这张表没有数据（data/coverage.py）：这几列的 0 不是真的是 0
|
||||
if isinstance(gap, dict) and gap.get("message"):
|
||||
lines.append(NO_DATA_QUERY_NOTE.format(message=gap["message"]))
|
||||
if not cells_allowed:
|
||||
return lines
|
||||
columns = [str(c) for c in snapshot.get("columns") or []][:_PROMPT_COLS]
|
||||
@@ -3838,8 +3860,13 @@ def _metric_prompt(catalog: dict[str, Any]) -> list[str]:
|
||||
if e.get("masked"):
|
||||
# 值取自遮罩列的一格（withhold_metrics）：写作者看到的就是「已遮罩」，引用了也只会交回重写
|
||||
lines.append(f"- [[{e['alias']}]] {e.get('name')} = {e.get('rendered')}（取自设置了遮罩的列，不要引用）")
|
||||
elif e.get("value") is None and e.get("undefined"):
|
||||
# 输入都齐、计算式自己算不出来（分母为 0）：照实写算不出来，不写成 0、持平
|
||||
lines.append(f"- [[{e['alias']}]] {e.get('name')}：这次算不出来（{UNDEFINED_REASON}），不要引用；"
|
||||
"照实写算不出来，不要写成 0 或持平" + (NO_DATA_METRIC_NOTE if e.get("no_data") else ""))
|
||||
elif e.get("value") is None:
|
||||
lines.append(f"- [[{e['alias']}]] {e.get('name')}：这次没有值，不要引用")
|
||||
lines.append(f"- [[{e['alias']}]] {e.get('name')}：这次没有值，不要引用"
|
||||
+ (NO_DATA_METRIC_NOTE if e.get("no_data") else ""))
|
||||
elif e.get("rendered") == MISSING:
|
||||
# 有值但按口径卡的格式显示不出来（太小、太大、不是有限的数）：引用了也是解析不了
|
||||
lines.append(f"- [[{e['alias']}]] {e.get('name')}：按口径卡的格式显示不出来，不要引用")
|
||||
@@ -3850,6 +3877,8 @@ def _metric_prompt(catalog: dict[str, Any]) -> list[str]:
|
||||
# 取数的 SQL 没通过检查（重复计算、存量跨期加总）：数是算错的可能很大，写作者引用时得交代
|
||||
if e.get("sql_check_failed"):
|
||||
caveat += "（所依据的查询未通过 SQL 检查，结果存疑：引用时须如实说明，不要当作确定的结论）"
|
||||
if e.get("no_data"):
|
||||
caveat += NO_DATA_METRIC_NOTE
|
||||
lines.append(f"- [[{e['alias']}]] {e.get('name')} = {e.get('rendered')}{caveat}")
|
||||
inputs = [e for e in catalog.values() if e.get("kind") == "input"]
|
||||
if inputs:
|
||||
|
||||
@@ -475,6 +475,15 @@ def incomplete_gaps(metrics: list[dict[str, Any]]) -> list[str]:
|
||||
for m in metrics if isinstance(m, dict) and m.get("incomplete")]
|
||||
|
||||
|
||||
def no_data_gaps(metrics: list[dict[str, Any]]) -> list[str]:
|
||||
"""契约收来的指标里，输入取自「查询区间里这张表没有数据」的列的（口径卡标了 no_data，data/coverage.py），每个一句缺口。
|
||||
|
||||
数回指得上口径卡、复算也一致，可那个 0 是因为这段时间表里根本没有数据，不是真的是 0：盖完整出具的章等于替「上周到访
|
||||
0 人」作保。原因用口径卡记下的那句（查询记下的区间和表里的范围）。"""
|
||||
return [f"指标「{m.get('name') or m.get('id')}」{m.get('no_data_reason') or '所依据的查询区间里这张表没有数据'}"
|
||||
for m in metrics if isinstance(m, dict) and m.get("no_data")]
|
||||
|
||||
|
||||
def sql_check_gaps(metrics: list[dict[str, Any]], cited: dict[str, list[str]] | None = None, *,
|
||||
label: Callable[[str, str | None], str | None] | None = None) -> list[str]:
|
||||
"""sql_check_findings 的那几句缺口（见下）。"""
|
||||
|
||||
@@ -69,6 +69,7 @@ from app.core.events import EventType
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.data import catalog_view
|
||||
from app.data.coverage import NO_DATA_SHORT, no_data_of
|
||||
from app.engine import deadline, recompute
|
||||
from app.engine.approval import read_decision
|
||||
from app.engine.context import NodeContext, NodeError, NodeRejected
|
||||
@@ -635,6 +636,10 @@ async def _query_input(item: dict[str, Any], sources: _Sources, aliases: dict[st
|
||||
hidden = await sources.hidden(artifact)
|
||||
if column.lower() in hidden:
|
||||
return {}, f"输入 {item['path']} 取自设置了遮罩的列，不能用于复算"
|
||||
if (gap := no_data_of(snap, column)) is not None:
|
||||
# 原值所在的那一格落在「查询区间里这张表没有数据」的列上（data/coverage.py）：那个 0 不代表真的是 0，盲算查到 0
|
||||
# 也说明不了一致，不白派一次盲算，按无法复算记
|
||||
return {}, f"{NO_DATA_SHORT}（输入 {item['path']}：{gap.get('message') or ''}）"
|
||||
value = cell_value(item.get("value"))
|
||||
if not _number(value):
|
||||
return {}, f"输入 {item['path']} 不是数值"
|
||||
@@ -1003,6 +1008,30 @@ def _settled(found: Mapping[str, Any]) -> bool:
|
||||
return found.get("limit") == "max_queries" and found.get("value") is not None
|
||||
|
||||
|
||||
#: 无法复算的原因里带上盲算交回的理由时最多截多少字（理由是渲染后的正文，已经过遮罩）
|
||||
RATIONALE_CHARS = 160
|
||||
|
||||
|
||||
def _rationale_tail(found: Mapping[str, Any]) -> str:
|
||||
"""盲算触顶、按无法复算记时（_limit_text 那一路），它交回的理由截一段接在原因后面：审批卡、缺口说明上看得到它查到了
|
||||
什么。没有理由的交回空串（原因照旧）。"""
|
||||
text = " ".join(str(found.get("rationale") or "").split())
|
||||
if not text:
|
||||
return ""
|
||||
return f"；盲算交回的理由：{text[:RATIONALE_CHARS]}{'…' if len(text) > RATIONALE_CHARS else ''}"
|
||||
|
||||
|
||||
def _no_data_cell(artifact: Any, column: Any, load: Callable[[str], Any]) -> dict[str, Any] | None:
|
||||
"""这一格所在的列「查询区间里这张表没有数据」（查询快照的 no_data，data/coverage.py）：那一条，不是为 None。"""
|
||||
if not isinstance(artifact, str) or not artifact or not isinstance(column, str):
|
||||
return None
|
||||
try:
|
||||
snapshot = load(artifact)
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
return no_data_of(snapshot, column)
|
||||
|
||||
|
||||
def _limit_text(limit: str | None, options: _Options) -> str:
|
||||
"""盲算为什么停下，接在「盲算」后面读：「盲算已达每项最多查询次数（6 次）」。"""
|
||||
if limit == "max_queries" and options.max_queries is not None:
|
||||
@@ -1027,12 +1056,17 @@ def _compare(row: dict[str, Any], metric: dict[str, Any], results: dict[str, Any
|
||||
row.update(status=UNCHECKED, reason="盲算没有执行")
|
||||
return
|
||||
if found.get("limit") and not _settled(found):
|
||||
row.update(status=UNCHECKED, reason="盲算" + _limit_text(found["limit"], options))
|
||||
row.update(status=UNCHECKED, reason="盲算" + _limit_text(found["limit"], options) + _rationale_tail(found))
|
||||
return
|
||||
if found.get("value") is None:
|
||||
row.update(status=UNCHECKED, reason=found.get("reason") or "盲算没有交回可用的数值",
|
||||
**{k: found[k] for k in ("cause", "run_seconds") if k in found}) # 运行时长上限将至（_blind）
|
||||
return
|
||||
if gap := _no_data_cell(found.get("artifact"), found.get("column"), load):
|
||||
# 盲算交回的那一格落在「查询区间里这张表没有数据」的列上（data/coverage.py）：那个 0 不代表真的是 0，拿它和原值
|
||||
# 比一致不一致都说明不了什么，按无法复算记
|
||||
row.update(status=UNCHECKED, reason=f"{NO_DATA_SHORT}（盲算查到的那一格：{gap.get('message') or ''}）")
|
||||
return
|
||||
values[inp["path"]] = found["value"]
|
||||
outcome = recompute.recompute(metric, recompute.Changes(inputs=values), recompute.Sources(
|
||||
load=load, outputs=outputs, masked=masks, queries=queries))
|
||||
|
||||
@@ -150,7 +150,7 @@ def _apply_contract(
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.engine.governance import board_only
|
||||
from app.engine.issuance import decide_tier, incomplete_gaps, sql_check_findings, trace_numbers
|
||||
from app.engine.issuance import decide_tier, incomplete_gaps, no_data_gaps, sql_check_findings, trace_numbers
|
||||
|
||||
nodes = state.get("nodes") or {}
|
||||
# 写了 report_from 就是引用模式：按报告文档逐段复核，不再按数值回指叙述
|
||||
@@ -191,6 +191,8 @@ def _apply_contract(
|
||||
gaps.append(f"指标来源未解析：{'、'.join(unresolved[:5])}")
|
||||
# 拿截断的查询结果整组算出来的指标：数回指得上，但算的只是取回的那一截，不能完整出具
|
||||
gaps.extend(incomplete_gaps(metrics))
|
||||
# 输入取自「查询区间里这张表没有数据」的列的指标：数回指得上、复算一致，但那个 0 不代表真的是 0，不能完整出具
|
||||
gaps.extend(no_data_gaps(metrics))
|
||||
# 来源查询没通过 SQL 检查（error 级：重复计算、存量跨期加总）：口径卡复算得一致，算的却是错数。
|
||||
# 报告直接引用的查询要等引用核对之后才知道，缺口放在这个位置，下面一起算
|
||||
sql_at = len(gaps)
|
||||
@@ -489,6 +491,8 @@ def _check_citations(
|
||||
"counted": governed and bool(unknown)}
|
||||
|
||||
out["queries"], out["aliases"] = _cited_queries(doc, catalog)
|
||||
# 报告直接引用了「查询区间里这张表没有数据」的列的格（查询快照的 no_data）：这些 0 不代表真的是 0，按缺口降档
|
||||
gaps.extend(_cited_no_data(doc, catalog))
|
||||
|
||||
# 老前端要用的逐个数字出处，从复核通过的数字片段拼出来
|
||||
for _, unit in iter_units(doc):
|
||||
@@ -854,6 +858,37 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
|
||||
}
|
||||
|
||||
|
||||
def _cited_no_data(doc: dict[str, Any], catalog: dict[str, Any]) -> list[str]:
|
||||
"""报告直接引用的格里，落在「查询区间里这张表没有数据」的列上的（查询快照的 no_data，data/coverage.py）：按列一句缺口
|
||||
「报告引用的 Q1「上周到访人数」：所依据的查询区间里这张表没有数据（…）」。读不出来的快照不算。"""
|
||||
from app.core.artifact_store import load
|
||||
from app.data.coverage import NO_DATA_SHORT, no_data_of
|
||||
from app.engine.evidence import iter_units
|
||||
|
||||
snapshots: dict[str, Any] = {}
|
||||
out: list[str] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for _, unit in iter_units(doc):
|
||||
for seg in unit.get("segments") or []:
|
||||
cite = seg.get("cite") or {}
|
||||
if cite.get("kind") != "cell" or cite.get("status") != "resolved" or not cite.get("alias"):
|
||||
continue
|
||||
alias = str(cite["alias"])
|
||||
column = str((cite.get("locator") or {}).get("column") or "")
|
||||
artifact = (catalog.get(alias) or {}).get("artifact")
|
||||
if not column or not isinstance(artifact, str) or (alias, column) in seen:
|
||||
continue
|
||||
seen.add((alias, column))
|
||||
if artifact not in snapshots:
|
||||
try:
|
||||
snapshots[artifact] = load(artifact)
|
||||
except (ValueError, OSError):
|
||||
snapshots[artifact] = None
|
||||
if (found := no_data_of(snapshots[artifact], column)) is not None:
|
||||
out.append(f"报告引用的 {alias}「{column}」：{NO_DATA_SHORT}（{found.get('message') or ''}）")
|
||||
return out
|
||||
|
||||
|
||||
def _cited_queries(doc: dict[str, Any], catalog: dict[str, Any]) -> tuple[dict[str, list[str]], dict[str, str]]:
|
||||
"""报告直接引用的查询：({查询快照 id: [引用的写法…]}, {目录里每个查询的快照 id: 编号})。
|
||||
|
||||
|
||||
@@ -82,6 +82,8 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
checked: dict[str, list[QueryProblem]] = {}
|
||||
#: 指标 id → 它没通过 SQL 检查的那几条来源查询。运行日志按查询归并（_emit_sql_check_logs）
|
||||
failed_by: dict[str, list[QueryProblem]] = {}
|
||||
#: 查询快照工件 id → 快照本身（读输入的格落在哪一列「查询区间里这张表没有数据」，_no_data_inputs）
|
||||
snapshots: dict[str, Any] = {}
|
||||
for definition in definitions:
|
||||
metric_id = str(definition.get("id") or "").strip()
|
||||
expr = str(definition.get("expression") or "").strip()
|
||||
@@ -159,6 +161,12 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
"sql_check_sources": [{"artifact": artifact, "codes": list(dict.fromkeys(
|
||||
code for p in found for code in p.codes))} for artifact, found in failed]}
|
||||
failed_by[metric_id] = [p for _, found in failed for p in found]
|
||||
# 输入取自「查询区间里这张表没有数据」的列（查询快照的 no_data，data/coverage.py）：值照算，标 no_data 并写明原因，
|
||||
# 出具契约据此降档（issuance.no_data_gaps）。按输入的快照 id + 列认：同一条查询里有数的列不受影响。转抄的指标
|
||||
# 那张卡标了的，这一个照样标
|
||||
no_data: dict[str, Any] = {}
|
||||
if lacking := _no_data_inputs(inputs, carried, snapshots):
|
||||
no_data = {"no_data": True, "no_data_reason": _no_data_reason(lacking), "no_data_sources": lacking}
|
||||
metrics.append(
|
||||
{
|
||||
"id": metric_id,
|
||||
@@ -175,10 +183,15 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
# 值是空的就是缺输入，不论 on_missing 是哪一种：fail 模式下直接取到空值的
|
||||
# 指标（原式就是 vars.x.y 而 y 不存在）以前也是照常产出一个 None
|
||||
"status": "ok" if value is not None else "missing_input",
|
||||
# 输入都齐、是计算式自己交回的空值（分母为 0 时写的 … if 分母 else None）：算不出来，不是缺输入。状态照旧是
|
||||
# missing_input（证据面板、推演、运行对比按状态认的地方不受影响），只多这一个键；有值的指标和以前一字不差
|
||||
**({"undefined": True} if value is None and not missing else {}),
|
||||
# 只在不完整时出现：完整的指标和以前一字不差，老运行复核时目录也对得上
|
||||
**incomplete,
|
||||
# 只在来源查询没通过 SQL 检查时出现，同上
|
||||
**sql_check,
|
||||
# 只在输入取自「查询区间里这张表没有数据」的列时出现，同上
|
||||
**no_data,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -188,11 +201,11 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
|
||||
# text 是给叙述节点 prompt 用的清单——它是叙述层唯一的数字来源
|
||||
lines = [
|
||||
(f"- {m['id']}（{m['name']}）= {m['value']}{m['unit']}" if m["value"] is not None
|
||||
else f"- {m['id']}（{m['name']}）= {MISSING}（缺少输入，无值）")
|
||||
f"- {m['id']}（{m['name']}）= {_listed(m)}"
|
||||
# 不写行数：叙述节点照抄进正文就是一个没有出处的数
|
||||
+ ("（不完整：基于被截断的查询结果计算）" if m.get("incomplete") else "")
|
||||
+ ("（存疑：所依据的查询未通过 SQL 检查）" if m.get("sql_check_failed") else "")
|
||||
+ (LISTED_NO_DATA if m.get("no_data") else "")
|
||||
for m in metrics
|
||||
]
|
||||
for m in metrics:
|
||||
@@ -202,6 +215,12 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
reason=m["incomplete_reason"], rows=[s.get("rows") for s in m.get("truncated_sources") or []],
|
||||
message=f"指标「{m['name']}」{m['incomplete_reason']}")
|
||||
_emit_sql_check_logs(metrics, failed_by, state, ctx)
|
||||
if lacking := [m for m in metrics if m.get("no_data")]:
|
||||
# 结构化字段给界面读：metrics / metric_names（受影响的指标）、reason（第一个指标的那句）、sources
|
||||
ctx.emit(EventType.LOG, level="warn", code="metric_no_data", metric=lacking[0]["id"],
|
||||
metrics=[m["id"] for m in lacking], metric_names=[m["name"] for m in lacking],
|
||||
reason=lacking[0]["no_data_reason"],
|
||||
message="指标" + "".join(f"「{m['name']}」" for m in lacking) + lacking[0]["no_data_reason"])
|
||||
card = {
|
||||
"kind": "metric_set",
|
||||
"caliber": caliber,
|
||||
@@ -443,6 +462,72 @@ def _rendered(value: Any, definition: dict[str, Any], display: dict[str, Any]) -
|
||||
return MISSING
|
||||
|
||||
|
||||
#: 清单里输入取自「查询区间里这张表没有数据」的列的指标后面那句：直接读清单的模型节点不能把 0 当成真的是 0
|
||||
LISTED_NO_DATA = "（这段时间表里没有数据：不要当成 0 解读）"
|
||||
#: 指标的 no_data_reason 打头那半句（后面接查询记下的说明）
|
||||
NO_DATA_REASON = "所依据的查询区间里这张表没有数据"
|
||||
|
||||
|
||||
def _no_data_inputs(inputs: list[dict[str, Any]], carried: list[dict[str, Any]],
|
||||
snapshots: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""这个指标的输入里，取自「查询区间里这张表没有数据」的列的：[{artifact, column, table, window, message}]。
|
||||
|
||||
认两处：输入的出处带着查询快照 id 和列（cell() 取的格、按位置取的格、Agent 经 cite_fields 交来的字段），快照里的
|
||||
no_data 记着这一列；转抄的指标那张卡标了 no_data 的（no_data_sources 照抄）。读不出来的快照不算。"""
|
||||
from app.data.coverage import no_data_of
|
||||
|
||||
out: list[dict[str, Any]] = []
|
||||
for item in inputs:
|
||||
artifact = item.get("artifact")
|
||||
locator = item.get("locator") if isinstance(item.get("locator"), dict) else {}
|
||||
columns = [locator.get("column")] if locator.get("column") is not None \
|
||||
else list((locator.get("columns") or {}).values()) if isinstance(locator.get("columns"), dict) else []
|
||||
if not isinstance(artifact, str) or not artifact or not columns:
|
||||
continue
|
||||
if artifact not in snapshots:
|
||||
try:
|
||||
snapshots[artifact] = artifact_store.load(artifact)
|
||||
except (ValueError, OSError):
|
||||
snapshots[artifact] = None
|
||||
for column in columns:
|
||||
found = no_data_of(snapshots[artifact], column)
|
||||
if found is None:
|
||||
continue
|
||||
entry = {"artifact": artifact, "column": str(column), "table": found.get("table"),
|
||||
"window": found.get("window"), "message": found.get("message")}
|
||||
if entry not in out:
|
||||
out.append(entry)
|
||||
for metric in carried:
|
||||
for entry in metric.get("no_data_sources") or []:
|
||||
if isinstance(entry, dict) and entry not in out:
|
||||
out.append(dict(entry))
|
||||
return out
|
||||
|
||||
|
||||
def _no_data_reason(lacking: list[dict[str, Any]]) -> str:
|
||||
"""「所依据的查询区间里这张表没有数据：「上周到访人数」查的是 … 」：出具缺口、运行日志、证据面板、写作目录都用这句。"""
|
||||
said = list(dict.fromkeys(str(e.get("message") or "") for e in lacking if e.get("message")))
|
||||
return NO_DATA_REASON + (f"：{'；'.join(said[:2])}" if said else "")
|
||||
|
||||
|
||||
#: 清单里没有值的两种说法：缺输入，和输入都齐、计算式自己算不出来（分母为 0）
|
||||
LISTED_MISSING = "（缺少输入，无值）"
|
||||
LISTED_UNDEFINED = "（算不出来：这组输入下计算式没有值，比如分母为 0）"
|
||||
#: 有值、按口径卡的小数位和格式显示不出来（太小、太大、不是有限的数）：不把原值拼出来，照实说
|
||||
LISTED_UNSHOWABLE = "（有值，但按设定的小数位和格式显示不出来）"
|
||||
|
||||
|
||||
def _listed(metric: dict[str, Any]) -> str:
|
||||
"""清单里这个指标的值：卡上按单位、小数位、格式渲染好的字（rendered，和报告里经 [[m:]] 渲染出来的一样）。以前拼原值
|
||||
（{value}{unit}），比率 0.0007 配 % 写成「0.0007%」，直接读清单的模型节点差 100 倍。"""
|
||||
if metric["value"] is None:
|
||||
return MISSING + (LISTED_UNDEFINED if metric.get("undefined") else LISTED_MISSING)
|
||||
rendered = str(metric.get("rendered") or "")
|
||||
if not rendered or rendered == MISSING:
|
||||
return MISSING + LISTED_UNSHOWABLE
|
||||
return rendered
|
||||
|
||||
|
||||
def _value_of(path: str, tctx: dict[str, Any]) -> Any:
|
||||
"""单独取一个引用的值。取不到（或取值本身出错）一律当作没有值。"""
|
||||
try:
|
||||
|
||||
@@ -56,6 +56,8 @@ from app.engine.evidence import (
|
||||
CLAIMS_RULE,
|
||||
ENTITY_KINDS,
|
||||
MARKER_RULES,
|
||||
NO_DATA_MARK,
|
||||
NO_DATA_RULE,
|
||||
SEE_AND_CODE_RULES,
|
||||
UNIT_RULE,
|
||||
StreamRenderer,
|
||||
@@ -651,6 +653,9 @@ def _messages(ctx: NodeContext, state: GraphState, writer: WriterCatalog, claims
|
||||
if writer.guarded:
|
||||
# 写作者看不全证据（目录有省略、有来源的结构化结论为空）：「没有查到」这类结论没有依据
|
||||
rules = f"{rules}\n{ABSENCE_RULE}"
|
||||
if NO_DATA_MARK in writer.text:
|
||||
# 有指标、查询列取自「查询区间里这张表没有数据」的列（data/coverage.py）：照实写没有数据，不写成 0、持平
|
||||
rules = f"{rules}\n{NO_DATA_RULE}"
|
||||
if claims in ("require_citation", CLAIMS_JUDGE):
|
||||
# 裁判只看写作者挂的依据：要判，就更得每句都挂上
|
||||
rules = f"{rules}\n{CLAIMS_RULE}" + (f"\n{judging.JUDGE_RULE}" if claims == CLAIMS_JUDGE else "")
|
||||
|
||||
@@ -609,7 +609,8 @@ _ANY = object()
|
||||
_CARD_PLAIN = frozenset({"kind", "caliber", "caliber_version", "artifact", "source"})
|
||||
#: 一个指标里不带值的字段（名称、单位、表达式、显示格式、状态这类）。值、渲染、输入、代入式都带着值
|
||||
_METRIC_PLAIN = frozenset({"id", "name", "unit", "expression", "decimals", "format", "status", "recompute_ok",
|
||||
"incomplete", "incomplete_reason", "sql_check_failed", "sql_check_reason"})
|
||||
"incomplete", "incomplete_reason", "sql_check_failed", "sql_check_reason", "undefined",
|
||||
"no_data", "no_data_reason", "no_data_sources"})
|
||||
#: 转抄的上游口径卡当时的产出读不出来（文件不在、与哈希不一致、不知道是谁写的）：转抄来的是不是遮罩值核对不了
|
||||
CARD_UNREADABLE = "转抄的口径卡结果无法读取，核对不了这个值是否取自设置了遮罩的列"
|
||||
|
||||
|
||||
@@ -137,11 +137,18 @@ WARN_CODES: dict[str, tuple[str, str]] = {
|
||||
# 核查 agent 的输出到了上限被截断（推理模型的思考也算在额度里）：没交成结果，由节点按无法复算、未能反查记缺口。
|
||||
# 同 verify_limit，是缺口不是断裂，记成它自己的 kind：说得清是输出额度的问题，换个模型就好
|
||||
"verify_truncated": ("verify_truncated", DEGRADED),
|
||||
# 查询区间里这张表没有数据（data/coverage.py）：查询结果整列是 0，口径卡的指标取自这些列。答案照常交付（出具契约另按
|
||||
# 缺口降档），是缺口不是断裂；但换一张有这段数据的表再建一次大概率就好，所以能给出重来的指示（REBUILD）
|
||||
"query_no_data": ("query_no_data", DEGRADED),
|
||||
"metric_no_data": ("metric_no_data", DEGRADED),
|
||||
}
|
||||
|
||||
# 这些 kind 重跑一次大概率能好：原因说得清，且不是「问题本身没问明白」。
|
||||
# 工具参数错了、步数不够、图没校验过，都属于改一下再来一遍就行的。
|
||||
RETRYABLE = {"step_limit", "tool_error", "empty_completion", "node_failed"}
|
||||
#: 是缺口（DEGRADED）、却值得重建一次的信号：查询区间里这张表没有数据——图选错了表，换一张有这段数据的表大概率就好。
|
||||
#: 答案照常交付，复核照样可以给出「按复核重来」的指示（should_retry、chat/turn.retry_after_review 都认）
|
||||
REBUILD = {"query_no_data", "metric_no_data"}
|
||||
|
||||
#: 串行工具模式下塞给未执行调用的占位说明。它漏到用户面前当过答案，
|
||||
#: 所以复核要认得出来。从 nodes.llm 导入会绕成循环，这里按内容识别。
|
||||
@@ -357,8 +364,8 @@ def worst(signals: Iterable[Signal]) -> str:
|
||||
|
||||
|
||||
def should_retry(signals: Iterable[Signal]) -> bool:
|
||||
"""这些信号里有没有「重跑一次大概率能好」的。"""
|
||||
return any(s.kind in RETRYABLE and s.severity == BROKEN for s in signals)
|
||||
"""这些信号里有没有「重跑一次大概率能好」的：严重的那几类（RETRYABLE），或者值得重建一次的缺口（REBUILD）。"""
|
||||
return any((s.kind in RETRYABLE and s.severity == BROKEN) or s.kind in REBUILD for s in signals)
|
||||
|
||||
|
||||
def check_numbers(original: str, revised: str) -> list[str]:
|
||||
|
||||
@@ -1464,6 +1464,40 @@ def _caliber_inputs(card: GraphNode, spec: GraphSpec) -> list[tuple[int, GraphNo
|
||||
return out
|
||||
|
||||
|
||||
def _is_zero(node: Any) -> bool:
|
||||
"""常量 0（0、0.0、-0、+0）；布尔不算。"""
|
||||
import ast
|
||||
|
||||
if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)):
|
||||
node = node.operand
|
||||
return isinstance(node, ast.Constant) and type(node.value) in (int, float) and node.value == 0
|
||||
|
||||
|
||||
def _divides(node: Any) -> bool:
|
||||
import ast
|
||||
|
||||
return any(isinstance(n, ast.BinOp) and isinstance(n.op, (ast.Div, ast.FloorDiv, ast.Mod)) for n in ast.walk(node))
|
||||
|
||||
|
||||
def zero_for_undefined(expression: str) -> bool:
|
||||
"""口径卡的表达式在算不出来时给了 0：某个条件表达式（x if c else y，嵌套的也算）一边有除法、另一边是常量 0——
|
||||
`round((a - b) / b, 4) if b else 0`、`0 if not b else a / b`。分母为 0、没有数据时给 0，报告里写成「0%」「持平」，
|
||||
读的人当成真的没变；应给 None（显示「—」）。按语法树认，解析不了的不算（另有表达式错误的报法）。"""
|
||||
import ast
|
||||
|
||||
from app.engine.expressions import ExpressionError, parse_expression
|
||||
|
||||
try:
|
||||
tree, _, _ = parse_expression(expression)
|
||||
except (ExpressionError, ValueError, TypeError):
|
||||
return False
|
||||
for n in ast.walk(tree):
|
||||
if isinstance(n, ast.IfExp) and ((_divides(n.body) and _is_zero(n.orelse))
|
||||
or (_divides(n.orelse) and _is_zero(n.body))):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def evidence_issues(spec: GraphSpec) -> list[ValidationIssue]:
|
||||
from app.engine.governance import UPGRADE_POLICIES, UPGRADE_POLICY_LABELS
|
||||
|
||||
@@ -1488,6 +1522,13 @@ def evidence_issues(spec: GraphSpec) -> list[ValidationIssue]:
|
||||
elif node.type == NodeType.METRICS:
|
||||
if cfg.get("caliber_from") not in (None, "", {}):
|
||||
out.extend(_caliber_from_issues(node, UPGRADE_POLICIES, UPGRADE_POLICY_LABELS, spec))
|
||||
else:
|
||||
for i, d in enumerate(cfg.get("metrics") or []):
|
||||
if isinstance(d, dict) and zero_for_undefined(str(d.get("expression") or "")):
|
||||
add(f"指标「{d.get('name') or d.get('id') or i + 1}」在算不出来时给了 0（条件表达式的一边是除法，"
|
||||
"另一边是 0）：分母为 0、没有数据时应给 None（报告里显示「—」，写作者照实说算不出来），"
|
||||
"写成 0 会被当成持平 / 0%。请改成「… if 分母 else None」",
|
||||
node, f"metrics[{i}].expression", code="metrics.zero_for_undefined")
|
||||
warned: set[str] = set()
|
||||
for i, producer in _caliber_inputs(node, spec):
|
||||
role = str(producer.config.get("evidence_role") or "compute")
|
||||
|
||||
@@ -696,6 +696,12 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False,
|
||||
# 快照和以前一字不差
|
||||
if checks := await _sql_checks(_checker(), sql):
|
||||
payload["checks"] = checks
|
||||
# 结果里有整列是 0 或空的数值列时，现查查询区间里这张表有没有数据（data/coverage.py）：没有就照实写进结果和
|
||||
# 快照（no_data，给人和模型看的 message），运行事件里记一条 query_no_data。没查出问题不加这个键，快照和以前一字不差
|
||||
if gaps := await _coverage(source, str(payload.get("sql") or sql), payload, mask):
|
||||
payload["no_data"] = gaps
|
||||
await _announce(ctx, EventType.LOG, level="warn", code="query_no_data", no_data=gaps,
|
||||
message="；".join(g["message"] for g in gaps))
|
||||
if outside and (unchecked := _unchecked_tables(sql, source, outside)):
|
||||
payload["unchecked_tables"] = unchecked
|
||||
# 表结构快照：这次查询时数据源的结构冻结下来，报告里写的表名、字段名按它核对。事后数据源
|
||||
@@ -776,6 +782,33 @@ def _unchecked_tables(sql: str, source: Any, outside: frozenset[str]) -> list[st
|
||||
if name_key(str(t).rsplit(".", 1)[-1]) in outside and data_catalog.resolve_table_name(cache, str(t)) is None]
|
||||
|
||||
|
||||
async def _coverage(source: Any, sql: str, payload: Mapping[str, Any], mask: list[str]) -> list[dict[str, Any]]:
|
||||
"""查询区间里这张表没有数据的那几列（data/coverage.result_gaps）。只在结果里有整列是 0 或空的数值列时才查库；遮罩列
|
||||
不探查；现查表的范围不成时退回剖析、取值观察记下的范围（上传表格的快照对不上就不用）。从不抛异常。"""
|
||||
from app.data import coverage
|
||||
|
||||
if not coverage.zero_columns(payload):
|
||||
return []
|
||||
|
||||
async def facts(table: str) -> Mapping[str, Any] | None:
|
||||
from app.data.catalog_profile_jobs import last_facts
|
||||
from app.db.base import SessionLocal
|
||||
|
||||
source_id = getattr(source, "id", None)
|
||||
if not source_id:
|
||||
return None
|
||||
name = data_catalog.resolve_table_name(getattr(source, "schema_cache", None), table) or table
|
||||
async with SessionLocal() as session:
|
||||
found = (await last_facts(session, str(source_id), [name])).get(name)
|
||||
snapshot = getattr(source, "snapshot_id", None)
|
||||
if found and snapshot and found.get("snapshot_id") != snapshot:
|
||||
# 上传表格：剖析时是另一版数据，范围对不上这次读的这一版
|
||||
return None
|
||||
return found
|
||||
|
||||
return await coverage.result_gaps(source, sql, payload, facts=facts, masked=mask)
|
||||
|
||||
|
||||
async def _sql_checks(checker: SqlChecker, sql: str) -> list[dict[str, Any]]:
|
||||
"""一条查询的 SQL 检查结果（as_dict 的形状）。在线程里跑、限时 SQL_CHECK_TIMEOUT_S：解析是纯计算，放在事件循环里
|
||||
会挡住同一进程里别的运行。出错、超时都返回空列表——检查不能让一次成功的查询变成失败。"""
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
"""助手搭图时就发现「查的区间里表里没有数据」（copilot_gen.GraphChecker.coverage、data/coverage.sql_gaps）。
|
||||
|
||||
部署上撞过：周报的取数节点用一张只存最新一天的表算上周、上上周，跑完才发现全是 0。现在：
|
||||
|
||||
- 自查：助手搭的图里工具节点写死的 SQL，按今天跑一遍覆盖检查（不跑原 SQL，只跑算界和现查范围的两条小查询，按表去重、
|
||||
限时）；完全不重叠的作为一条问题交回模型改一轮（换有这段数据的表，或者在说明里照实告诉用户）；修正后仍在的不挡运行，
|
||||
作为提醒留在检查结果里；
|
||||
- look / db_schema 的剖析信息：日期范围最早和最晚是同一天时加一句「可能每天整表重灌：算历史周期请换有历史的表」；
|
||||
- 助手提示：写按周期取数的 SQL 前先看表的日期范围。
|
||||
|
||||
库是现造的 SQLite 文件，表名、列名、数字都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessageChunk
|
||||
|
||||
from app.data import introspect
|
||||
from app.data.engine import engines
|
||||
from app.data.schema_view import PROFILE_HINT_PROMPT, profile_lines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from tests.test_copilot_answer_ops import _sse, _use, client # noqa: F401
|
||||
|
||||
NO_DATA_SQL = "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012' AND event_date < '20261019'"
|
||||
HAS_DATA_SQL = "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20200101'"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def plant(tmp_path):
|
||||
path = tmp_path / f"{uuid.uuid4().hex[:8]}.db"
|
||||
with sqlite3.connect(path) as db:
|
||||
db.executescript("CREATE TABLE gate_events (event_date TEXT, member_id INTEGER);"
|
||||
"INSERT INTO gate_events VALUES ('20200101', 1), ('20200101', 2);")
|
||||
name = f"plant_{uuid.uuid4().hex[:6]}"
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True)
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
row.schema_cache = await introspect.introspect(row)
|
||||
await session.commit()
|
||||
source_id = row.id
|
||||
async with SessionLocal() as session:
|
||||
source = await session.get(DataSource, source_id)
|
||||
yield name, source
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
def _node(nid: str, ntype: str, config: dict) -> dict:
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||||
|
||||
|
||||
def _graph(name: str, sql: str) -> tuple[dict, list]:
|
||||
nodes = {"start": _node("start", "input", {"fields": [{"name": "question"}]}),
|
||||
"fetch": _node("fetch", "tool", {"tool": f"db_query__{name}", "args": {"sql": sql}}),
|
||||
"out": _node("out", "output", {"fields": [{"name": "n", "value": "{{ nodes.fetch }}"}]})}
|
||||
return nodes, [{"source": "start", "target": "fetch"}, {"source": "fetch", "target": "out"}]
|
||||
|
||||
|
||||
async def test_the_checker_returns_it_once_then_keeps_it_as_a_warning(plant):
|
||||
from app.api.copilot_gen import GraphApplier, GraphChecker
|
||||
|
||||
name, source = plant
|
||||
nodes, edges = _graph(name, NO_DATA_SQL)
|
||||
checker = GraphChecker(GraphApplier(nodes, edges), sources=[source])
|
||||
[issue] = [i for i in await checker.issues() if i["code"] == "sql_no_data"]
|
||||
assert issue["level"] == "error" and issue["node_id"] == "fetch" and issue["field"] == "args.sql"
|
||||
assert "查的区间是 2026-10-12 到 2026-10-18，但 gate_events 的 event_date 只有 2020-01-01 这一天的数据" \
|
||||
in issue["message"]
|
||||
assert "换有这段数据的表" in issue["for_model"] and "explanation" in issue["for_model"]
|
||||
# 交回模型改过一轮：之后不再挡；收尾时作为提醒
|
||||
assert not [i for i in await checker.issues() if i["code"] == "sql_no_data"]
|
||||
[warning] = await checker.coverage(level="warning")
|
||||
assert warning["level"] == "warning" and warning["code"] == "sql_no_data"
|
||||
|
||||
|
||||
async def test_a_window_with_data_and_templated_sql_are_left_alone(plant):
|
||||
from app.api.copilot_gen import GraphApplier, GraphChecker
|
||||
|
||||
name, source = plant
|
||||
for sql in (HAS_DATA_SQL, "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '{{ input.day }}'"):
|
||||
nodes, edges = _graph(name, sql)
|
||||
checker = GraphChecker(GraphApplier(nodes, edges), sources=[source])
|
||||
assert not [i for i in await checker.issues() if i["code"] == "sql_no_data"], sql
|
||||
|
||||
|
||||
async def test_the_stream_asks_for_one_repair_then_runs_with_a_warning(client, monkeypatch, plant): # noqa: F811
|
||||
name, _ = plant
|
||||
nodes, edges = _graph(name, NO_DATA_SQL)
|
||||
build = [{"op": "plan", "summary": "数一数"},
|
||||
*({"op": "add_node", "node": {"id": n["id"], "type": n["type"], "label": n["id"],
|
||||
"config": n["data"]["config"]}} for n in nodes.values()),
|
||||
*({"op": "add_edge", "edge": e} for e in edges), {"op": "done", "explanation": "数一数", "run": True}]
|
||||
keep = [{"op": "done", "explanation": "gate_events 只有 2020-01-01 的数据，这段时间没有数据"}]
|
||||
seen: list[str] = []
|
||||
|
||||
class Scripted:
|
||||
async def astream(self, messages):
|
||||
human = next(text for role, text in messages if role == "human")
|
||||
seen.append(human)
|
||||
for op in (keep if "上线前自查" in human else build):
|
||||
yield AIMessageChunk(content=json.dumps(op, ensure_ascii=False) + "\n")
|
||||
|
||||
_use(monkeypatch, Scripted())
|
||||
got = await _sse(client, {"instruction": "上周到访多少次", "intent": "answer"})
|
||||
[repairing] = [op for op in got if op["op"] == "check" and op["status"] == "repairing"]
|
||||
assert [i["code"] for i in repairing["issues"]] == ["sql_no_data"]
|
||||
assert any("上线前自查" in h and "换有这段数据的表" in h for h in seen), "交回模型改一轮"
|
||||
[passed] = [op for op in got if op["op"] == "check" and op["status"] == "passed"]
|
||||
assert [w["code"] for w in passed["warnings"]] == ["sql_no_data"]
|
||||
[final] = [op for op in got if op["op"] == "final"]
|
||||
assert final["next"] == "run", "修正后仍在的不挡运行"
|
||||
assert any(i["code"] == "sql_no_data" and i["level"] == "warning" for i in final["issues"])
|
||||
|
||||
|
||||
def test_a_one_day_profile_says_so():
|
||||
facts = {"at": "2026-10-18T02:00:00+00:00", "date_ranges": [{"column": "event_date", "min": "20261017",
|
||||
"max": "20261017"}]}
|
||||
lines = profile_lines(facts, set(), None)
|
||||
assert any("剖析时这张表的 event_date 只有一天（2026-10-17）的数据，可能每天整表重灌：算历史周期请换有历史的表" in ln
|
||||
for ln in lines), lines
|
||||
assert lines[-1].strip() == PROFILE_HINT_PROMPT
|
||||
wide = {**facts, "date_ranges": [{"column": "event_date", "min": "20260101", "max": "20261017"}]}
|
||||
assert not any("整表重灌" in ln for ln in profile_lines(wide, set(), None))
|
||||
|
||||
|
||||
def test_the_copilot_is_told_to_look_at_the_dates_first():
|
||||
from app.api.copilot import NODE_REFERENCE
|
||||
|
||||
assert "先看表的日期范围" in NODE_REFERENCE and "只有一天的数据" in NODE_REFERENCE
|
||||
@@ -271,9 +271,11 @@ async def test_two_metrics_using_one_cell_are_checked_once(monkeypatch, shop):
|
||||
|
||||
|
||||
async def test_hitting_a_limit_counts_as_unchecked(monkeypatch, shop):
|
||||
"""查询次数用完了还没查到要交的那一格：按无法复算计入缺口；一个指标都没核成，产出是 unchecked，不是 agreed。"""
|
||||
"""查询次数用完了还没查到要交的那一格：按无法复算计入缺口；一个指标都没核成，产出是 unchecked，不是 agreed。
|
||||
盲算交回的理由截一段带进原因：审批卡、缺口说明上看得到它查到了什么。"""
|
||||
name, _ = shop
|
||||
Blind(monkeypatch, name, {"销售额": ([BLIND_TOTAL, RIGHT], "gmv")})
|
||||
Blind(monkeypatch, name, {"销售额": ([BLIND_TOTAL, RIGHT], "gmv")},
|
||||
rationale="只查到了不含退款的合计，还没来得及按支付日核对。")
|
||||
run = await start(graph(name, RIGHT), options={"verify": {"dual_check": {"max_queries": 1}}})
|
||||
assert run.status == "succeeded", run.error
|
||||
evs = await events(run.id)
|
||||
@@ -281,15 +283,16 @@ async def test_hitting_a_limit_counts_as_unchecked(monkeypatch, shop):
|
||||
out = dual_output(run, evs)
|
||||
assert out["status"] == "unchecked" and out["metrics"][0]["status"] == "unchecked"
|
||||
assert out["counts"] == {"agreed": 0, "disagreed": 0, "unchecked": 1}
|
||||
assert out["issuance_gaps"] == ["指标「销售额」无法复算：盲算已达每项最多查询次数（1 次）"]
|
||||
reason = "盲算已达每项最多查询次数（1 次）；盲算交回的理由：只查到了不含退款的合计，还没来得及按支付日核对。"
|
||||
assert out["issuance_gaps"] == [f"指标「销售额」无法复算：{reason}"]
|
||||
[said] = logs(evs, "dual_check_unchecked")
|
||||
assert said["metrics"] == [{"id": "gmv", "name": "销售额", "reason": "盲算已达每项最多查询次数（1 次）"}]
|
||||
assert said["metrics"] == [{"id": "gmv", "name": "销售额", "reason": reason}]
|
||||
assert not logs(evs, "dual_check_agreed")
|
||||
assert not [e for e in evs if e.type == "human.requested"]
|
||||
[finished] = [e for e in evs if e.type == "node.finished" and e.node_id == "dual"]
|
||||
assert finished.data["evidence"][0]["status"] == "unchecked"
|
||||
assert run.output["_issuance"]["tier"] == "degraded"
|
||||
assert run.output["_issuance"]["gaps"] == ["双盲复算「复核销售额」：指标「销售额」无法复算：盲算已达每项最多查询次数（1 次）"]
|
||||
assert run.output["_issuance"]["gaps"] == [f"双盲复算「复核销售额」：指标「销售额」无法复算：{reason}"]
|
||||
|
||||
|
||||
async def test_an_answer_on_the_last_allowed_query_still_counts(monkeypatch, shop):
|
||||
@@ -1575,3 +1578,41 @@ async def test_the_report_catalog_leaves_out_the_blind_queries(monkeypatch, shop
|
||||
if e.type == "tool.end" and e.node_id == "dual" and e.data.get("query_artifact")]
|
||||
ledger = [entry for e in evs if e.type == "node.finished" for entry in e.data.get("evidence") or []]
|
||||
assert blind_queries and not any(entry.get("artifact") in blind_queries for entry in ledger)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 查询区间里这张表没有数据（data/coverage.py）：不判一致，按无法复算记
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: 订单表只有 2026 年的数：2030 年那一周表里没有数据，COUNT 是 0
|
||||
EMPTY_WEEK = "BETWEEN '2030-01-07' AND '2030-01-13'"
|
||||
NO_ORDERS = f"SELECT COUNT(*) AS orders FROM orders WHERE status <> 'refunded' AND paid_date {EMPTY_WEEK}"
|
||||
ORDERS = {"id": "orders", "name": "订单数", "unit": "单", "expression": "cell(nodes.fetch, 0, 'orders')"}
|
||||
|
||||
|
||||
async def test_an_original_cell_without_data_is_not_checked(monkeypatch, shop):
|
||||
"""原值所在的格子落在 no_data 的列上：那个 0 不代表真的是 0，盲算查到 0 也说明不了一致。不派盲算，按无法复算记。"""
|
||||
name, _ = shop
|
||||
blind = Blind(monkeypatch, name, {"订单数": (NO_ORDERS, "orders")})
|
||||
run = await start(graph(name, NO_ORDERS, metrics=[ORDERS], dual={"definitions": {"orders": "已支付订单数"}}))
|
||||
assert run.status == "succeeded", run.error
|
||||
evs = await events(run.id)
|
||||
out = dual_output(run, evs)
|
||||
[row] = out["metrics"]
|
||||
assert row["status"] == "unchecked" and row["reason"].startswith("查询区间里这张表没有数据（输入 cell(nodes.fetch")
|
||||
assert "2030-01-07 到 2030-01-13" in row["reason"]
|
||||
assert "step" not in blind.calls, "原值就没有数据：不白派一次盲算"
|
||||
assert out["counts"] == {"agreed": 0, "disagreed": 0, "unchecked": 1}
|
||||
|
||||
|
||||
async def test_a_blind_cell_without_data_is_not_agreed(monkeypatch, shop):
|
||||
"""盲算交回的格子落在 no_data 的列上（它查了一段表里没有数据的区间）：不判一致，按无法复算记。"""
|
||||
name, _ = shop
|
||||
buyers = {"id": "orders", "name": "订单数", "unit": "单", "expression": "cell(nodes.fetch, 0, 'orders')"}
|
||||
original = f"SELECT COUNT(*) AS orders FROM orders WHERE status <> 'refunded' AND paid_date {WEEK}"
|
||||
Blind(monkeypatch, name, {"订单数": (NO_ORDERS.replace("AS orders", "AS n"), "n")})
|
||||
run = await start(graph(name, original, metrics=[buyers], dual={"definitions": {"orders": "已支付订单数"}}))
|
||||
assert run.status == "succeeded", run.error
|
||||
[row] = dual_output(run, await events(run.id))["metrics"]
|
||||
assert row["status"] == "unchecked", row
|
||||
assert row["reason"].startswith("查询区间里这张表没有数据（盲算查到的那一格：「n」查的是 2030-01-07 到 2030-01-13")
|
||||
|
||||
@@ -179,7 +179,8 @@ async def test_each_metric_records_its_inputs_substitution_and_rendering(engine_
|
||||
# 旧字段一个不少，text 照旧——模板 ⑧ 的叙述 prompt 靠它
|
||||
assert {"id", "name", "unit", "value", "expression"} <= set(wow)
|
||||
output = row.output["清单"]
|
||||
assert "- wow（环比增幅）= 8.7%" in output and "- orders（订单数）= 1234单" in output
|
||||
# 清单按卡上渲染好的字写（和报告里 [[m:]] 一样按单位、小数位、千分位）：叙述节点抄的就是报告上会出现的写法
|
||||
assert "- wow（环比增幅）= 8.7%" in output and "- orders（订单数）= 1,234单" in output
|
||||
|
||||
|
||||
async def test_metric_set_artifact_is_in_the_ledger_and_node_finished(engine_up):
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
"""口径卡的输入取自「查询区间里这张表没有数据」的列时，指标照实标出来（no_data），出具按缺口降档，复核给出重来的指示。
|
||||
|
||||
部署上撞过：一张周报的到访事件表只存最新一天，6 列全是 0，口径卡里 10 个指标是 0 或 0%，周报照写「上周到访人数为 0 人」。
|
||||
查询工具现在会在结果和快照里记 no_data（data/coverage.py）；这里守它往下走的几步：
|
||||
|
||||
- 口径卡按输入的快照 id + 列认（按列：同一条查询里有数的列不受影响），给指标标 no_data、写明原因；清单文字注明；
|
||||
运行日志一条 metric_no_data；
|
||||
- 有出具契约的成果按缺口降档；
|
||||
- 复核把 metric_no_data / query_no_data 记成缺口（DEGRADED），并给出「按复核重来」的指示：换一张有这段数据的表再建一次，
|
||||
大概率就好。
|
||||
|
||||
库是现造的 SQLite 文件，表名、列名、数字都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource, Run, RunEvent
|
||||
from app.engine.runner import run_manager
|
||||
|
||||
WINDOW = "{c} >= strftime('%Y%m%d', 'now', '-{a} days') AND {c} < strftime('%Y%m%d', 'now', '-{b} days')"
|
||||
KPI_SQL = ("SELECT (SELECT SUM(pv) FROM daily_pv WHERE day >= date('now', '-14 days')) AS 客流PV, "
|
||||
"(SELECT COUNT(DISTINCT member_id) FROM gate_events WHERE " + WINDOW.format(c="event_date", a=14, b=7)
|
||||
+ ") AS 上周到访人数, (SELECT COUNT(DISTINCT member_id) FROM gate_events WHERE "
|
||||
+ WINDOW.format(c="event_date", a=21, b=14) + ") AS 上上周到访人数")
|
||||
WOW = ("round((cell(nodes.q_kpi, 0, '上周到访人数') - cell(nodes.q_kpi, 0, '上上周到访人数')) "
|
||||
"/ cell(nodes.q_kpi, 0, '上上周到访人数'), 4) if cell(nodes.q_kpi, 0, '上上周到访人数') else None")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up():
|
||||
await run_manager.setup()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def plant(tmp_path):
|
||||
path = tmp_path / f"{uuid.uuid4().hex[:8]}.db"
|
||||
with sqlite3.connect(path) as db:
|
||||
db.executescript("CREATE TABLE gate_events (event_date TEXT, member_id INTEGER);"
|
||||
"INSERT INTO gate_events VALUES ('20200101', 1), ('20200101', 2);"
|
||||
"CREATE TABLE daily_pv (day TEXT, pv INTEGER);")
|
||||
for back in range(30):
|
||||
db.execute("INSERT INTO daily_pv VALUES (date('now', ?), 100)", (f"-{back} days",))
|
||||
name = f"plant_{uuid.uuid4().hex[:6]}"
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
source_id = row.id
|
||||
yield name
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
def node(nid, ntype, label=None, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}}
|
||||
|
||||
|
||||
def board(source: str) -> dict:
|
||||
nodes = [
|
||||
node("start", "input", "开始"),
|
||||
node("q_kpi", "tool", "取数", tool=f"db_query__{source}", args={"sql": KPI_SQL}),
|
||||
node("card", "metrics", "周报口径卡", caliber="周报口径", caliber_version="v1", metrics=[
|
||||
{"id": "pv", "name": "客流PV", "unit": "次", "expression": "cell(nodes.q_kpi, 0, '客流PV')"},
|
||||
{"id": "visitors", "name": "上周到访人数", "unit": "人", "expression": "cell(nodes.q_kpi, 0, '上周到访人数')"},
|
||||
{"id": "wow", "name": "到访周环比", "decimals": 4, "format": "percent_of_ratio", "expression": WOW},
|
||||
]),
|
||||
node("out", "output", "成果", fields=[{"name": "指标", "value": "{{ nodes.card.text }}"}],
|
||||
contract={"metrics_from": ["card"], "required": ["pv", "visitors"]}),
|
||||
]
|
||||
return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||||
|
||||
|
||||
async def finished(graph: dict) -> tuple[Run, list[RunEvent]]:
|
||||
run = await run_manager.start(graph=graph, input_payload={})
|
||||
for _ in range(400):
|
||||
await asyncio.sleep(0.05)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run.id)
|
||||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||||
assert row.status == "succeeded", row.error
|
||||
async with SessionLocal() as session:
|
||||
events = list((await session.execute(
|
||||
select(RunEvent).where(RunEvent.run_id == run.id).order_by(RunEvent.seq))).scalars())
|
||||
return row, events
|
||||
raise AssertionError(f"运行 {run.id} 没跑完：{row.status}")
|
||||
|
||||
|
||||
def _card(events: list[RunEvent]) -> dict:
|
||||
[event] = [e for e in events if e.type == "node.finished" and e.node_id == "card"]
|
||||
[entry] = event.data["evidence"]
|
||||
return artifact_store.load(entry["artifact"])
|
||||
|
||||
|
||||
async def test_metrics_fed_by_columns_without_data_are_flagged(plant):
|
||||
row, events = await finished(board(plant))
|
||||
metrics = {m["id"]: m for m in _card(events)["metrics"]}
|
||||
visitors, wow, pv = metrics["visitors"], metrics["wow"], metrics["pv"]
|
||||
assert visitors["value"] == 0 and visitors["no_data"] is True
|
||||
assert visitors["no_data_reason"].startswith("所依据的查询区间里这张表没有数据：")
|
||||
assert "gate_events" in visitors["no_data_reason"]
|
||||
[source] = visitors["no_data_sources"]
|
||||
assert source["column"] == "上周到访人数" and source["table"] == "gate_events" and source["artifact"]
|
||||
# 周环比两个输入都在没有数据的列上：照样标；它本身算不出来（分母为 0），两件事都照实记
|
||||
assert wow["no_data"] is True and wow["undefined"] is True and wow["value"] is None
|
||||
# 同一条查询里有数的列（客流 PV）不受影响
|
||||
assert "no_data" not in pv and pv["value"] > 0
|
||||
lines = row.output["指标"].split("\n")
|
||||
assert "- visitors（上周到访人数）= 0人（这段时间表里没有数据：不要当成 0 解读）" in lines
|
||||
assert not any("pv（客流PV）" in ln and "没有数据" in ln for ln in lines)
|
||||
# 运行日志：查询一条 query_no_data，口径卡一条 metric_no_data（受影响的指标都列出来）
|
||||
codes = [(e.data or {}).get("code") for e in events if e.type == "log"]
|
||||
assert "query_no_data" in codes
|
||||
[logged] = [e.data for e in events if e.type == "log" and (e.data or {}).get("code") == "metric_no_data"]
|
||||
assert logged["level"] == "warn" and logged["metrics"] == ["visitors", "wow"]
|
||||
|
||||
|
||||
async def test_the_contract_degrades_on_a_metric_without_data(plant):
|
||||
row, _ = await finished(board(plant))
|
||||
issuance = row.output["_issuance"]
|
||||
assert issuance["tier"] == "degraded"
|
||||
gaps = [g for g in issuance["gaps"] if "没有数据" in g]
|
||||
assert any("上周到访人数" in g for g in gaps) and any("到访周环比" in g for g in gaps), issuance["gaps"]
|
||||
|
||||
|
||||
async def test_review_marks_it_and_asks_for_a_rebuild(plant):
|
||||
from app.chat import turn
|
||||
from app.engine import review as rv
|
||||
|
||||
row, events = await finished(board(plant))
|
||||
signals = rv.scan(events, row.output)
|
||||
kinds = {s.kind: s.severity for s in signals}
|
||||
assert kinds.get("metric_no_data") == rv.DEGRADED and kinds.get("query_no_data") == rv.DEGRADED
|
||||
assert rv.should_retry(signals), "换一张有这段数据的表再建一次大概率就好"
|
||||
result = rv.ReviewResult(verdict="annotated", note="…", retry=True, signals=signals, severity=rv.worst(signals))
|
||||
assert turn.retry_after_review(result.as_dict(), 0) and not turn.retry_after_review(result.as_dict(), 1)
|
||||
# 别的降级信号照旧不重来
|
||||
other = [rv.Signal(kind="node_retry", detail="重试过一次", severity=rv.DEGRADED)]
|
||||
assert not rv.should_retry(other)
|
||||
assert not turn.retry_after_review(rv.ReviewResult(verdict="annotated", retry=True, signals=other,
|
||||
severity=rv.DEGRADED).as_dict(), 0)
|
||||
|
||||
|
||||
async def test_the_writer_is_told_and_given_the_rule(plant):
|
||||
"""写作目录里这样的指标、查询格子里这样的列都写清楚；写作规则加一句。"""
|
||||
from app.engine.evidence import NO_DATA_RULE, build_catalog, writer_catalog
|
||||
|
||||
row, events = await finished(board(plant))
|
||||
card = _card(events)
|
||||
[query] = [e for e in events if e.type == "node.finished" and e.node_id == "q_kpi"]
|
||||
ledger = [*query.data["evidence"], *[e for e in events if e.type == "node.finished"
|
||||
and e.node_id == "card"][0].data["evidence"]]
|
||||
catalog = build_catalog(nodes={"card": {**card, "artifact": ledger[-1]["artifact"]}}, ledger=ledger, inputs={})
|
||||
assert catalog["m:visitors"]["no_data"] is True and "no_data" not in catalog["m:pv"]
|
||||
text = writer_catalog(catalog).text
|
||||
visitors = next(ln for ln in text.splitlines() if "[[m:visitors]]" in ln)
|
||||
assert "这段时间表里没有数据" in visitors and "不要当成 0 解读" in visitors
|
||||
assert "这段时间表里没有数据" not in next(ln for ln in text.splitlines() if "[[m:pv]]" in ln)
|
||||
assert "（注意：「上周到访人数」" in text and "引用这几列时照实写这段时间表里没有数据" in text
|
||||
assert "照实写这段时间表里没有数据" in NO_DATA_RULE and "0%" in NO_DATA_RULE
|
||||
|
||||
|
||||
async def test_the_report_prompt_carries_the_rule(plant, monkeypatch):
|
||||
from app.engine.evidence import NO_DATA_RULE
|
||||
from tests.test_report_node import script
|
||||
|
||||
seen = script(monkeypatch, "上周到访人数这段时间表里没有数据。")
|
||||
graph = board(plant)
|
||||
out = graph["nodes"].pop()
|
||||
graph["nodes"] += [node("write", "report", "撰写", instructions="写一段周报"),
|
||||
{**out, "data": {**out["data"], "config": {"fields": [{"name": "答案",
|
||||
"value": "{{ nodes.write.text }}"}]}}}]
|
||||
graph["edges"] = [{"source": a["id"], "target": b["id"]} for a, b in zip(graph["nodes"], graph["nodes"][1:])]
|
||||
await finished(graph)
|
||||
prompt = "\n".join(str(m.content) for m in seen[0])
|
||||
assert NO_DATA_RULE in prompt and "这段时间表里没有数据" in prompt
|
||||
|
||||
|
||||
def test_an_ungated_result_says_its_gaps_on_the_stamp():
|
||||
"""成果没配出具契约时，实例页的印章照实写这些缺口（配了契约的已经按缺口降档）。"""
|
||||
from app.apps.resolve import ungated_notes
|
||||
|
||||
cards = [{"metrics": [{"id": "visitors", "name": "上周到访人数", "no_data": True}, {"id": "pv", "name": "客流PV"}]}]
|
||||
dual = {"issuance_gaps": ["指标「销售额」无法复算：盲算已达每项最多查询次数（1 次）"]}
|
||||
notes = ungated_notes(cards, dual)
|
||||
assert notes[0].startswith("指标「上周到访人数」所依据的查询区间里表里没有数据") and "没有配出具契约" in notes[0]
|
||||
assert notes[1].startswith("双盲复算有缺口：指标「销售额」无法复算")
|
||||
assert ungated_notes([{"metrics": [{"id": "pv", "name": "客流PV"}]}], None) == []
|
||||
@@ -0,0 +1,183 @@
|
||||
"""口径卡：分母为 0 时算不出来就是空值，和「缺少输入」分开说；指标清单按格式渲染。
|
||||
|
||||
部署上撞过：一份周报的口径卡全写成 `round((a - b) / b, 4) if b else 0`，两周都没有数据时周环比、复访率写成 0 / 0%，
|
||||
读起来像「持平」。现在：
|
||||
|
||||
- 助手的节点手册写明分母可能为 0 时写 `… if 分母 else None`，不要写 else 0；画布校验对「条件表达式一边有除法、另一边是
|
||||
常量 0」给提醒（metrics.zero_for_undefined，按语法树认，嵌套的也认）；
|
||||
- 输入都齐、是计算式自己交回空值的指标多记 undefined: true（状态仍是 missing_input，老运行、按状态认的地方不受影响），
|
||||
清单、写作目录、引用解析不了的原因、运行对比都说「算不出来」，不说「缺少输入」；
|
||||
- 指标清单（{{ nodes.卡.text }}）按卡上渲染好的字写：比率 0.0007 显示成 0.07%，不再拼原值写成「0.0007%」；值不为空
|
||||
却显示不出来的照实说。
|
||||
|
||||
数据、口径都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Run, RunEvent
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import GraphSpec, validate_graph
|
||||
|
||||
KPI = "{'cur': 0, 'prev': 0, 'visits': 1200, 'repeat': 9, 'gmv': 45678.5, 'tiny': 0.000000001}"
|
||||
METRICS = [
|
||||
{"id": "wow", "name": "周环比", "decimals": 4, "format": "percent_of_ratio",
|
||||
"expression": "round((vars.kpi.cur - vars.kpi.prev) / vars.kpi.prev, 4) if vars.kpi.prev else None"},
|
||||
{"id": "lost", "name": "缺一项", "expression": "vars.kpi.nothing"},
|
||||
{"id": "rate", "name": "复访率", "unit": "%", "decimals": 4, "format": "percent_of_ratio",
|
||||
"expression": "round(vars.kpi.repeat / vars.kpi.visits, 4)"},
|
||||
{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"},
|
||||
{"id": "tiny", "name": "很小的数", "decimals": 2, "expression": "vars.kpi.tiny"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up():
|
||||
await run_manager.setup()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||||
|
||||
|
||||
def graph(metrics=METRICS, kpi=KPI):
|
||||
nodes = [node("start", "input"), node("fetch", "transform", mode="expression", expression=kpi, assign_to="kpi"),
|
||||
node("card", "metrics", caliber="周报口径", caliber_version="v1", metrics=metrics),
|
||||
node("out", "output", fields=[{"name": "清单", "value": "{{ nodes.card.text }}"}])]
|
||||
return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||||
|
||||
|
||||
async def _card() -> tuple[Run, dict]:
|
||||
run = await run_manager.start(graph=graph(), input_payload={})
|
||||
for _ in range(300):
|
||||
await asyncio.sleep(0.05)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run.id)
|
||||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||||
break
|
||||
assert row.status == "succeeded", row.error
|
||||
async with SessionLocal() as session:
|
||||
[event] = (await session.execute(select(RunEvent).where(
|
||||
RunEvent.run_id == run.id, RunEvent.type == "node.finished", RunEvent.node_id == "card"))).scalars()
|
||||
[entry] = event.data["evidence"]
|
||||
return row, artifact_store.load(entry["artifact"])
|
||||
|
||||
|
||||
async def test_undefined_is_told_apart_from_a_missing_input():
|
||||
row, card = await _card()
|
||||
by_id = {m["id"]: m for m in card["metrics"]}
|
||||
wow, lost = by_id["wow"], by_id["lost"]
|
||||
# 状态都是 missing_input（按状态认的地方照旧）；算不出来的多一个 undefined
|
||||
assert wow["value"] is None and wow["status"] == "missing_input" and wow["undefined"] is True
|
||||
assert lost["value"] is None and lost["status"] == "missing_input" and "undefined" not in lost
|
||||
# 有值的指标没有这个键：卡和以前一字不差
|
||||
assert all("undefined" not in by_id[k] for k in ("rate", "gmv", "tiny"))
|
||||
lines = row.output["清单"].split("\n")
|
||||
assert "- wow（周环比）= —（算不出来：这组输入下计算式没有值，比如分母为 0）" in lines
|
||||
assert "- lost（缺一项）= —（缺少输入，无值）" in lines
|
||||
|
||||
|
||||
async def test_the_list_shows_the_rendered_value():
|
||||
row, _ = await _card()
|
||||
lines = row.output["清单"].split("\n")
|
||||
assert "- rate（复访率）= 0.75%" in lines, lines # 不是「0.0075%」
|
||||
assert "- gmv（销售额）= 45,678.5元" in lines
|
||||
# 卡先按小数位取整，再按格式渲染：很小的数取整成 0 就显示 0.00（和报告里一样），不拼原值
|
||||
assert "- tiny（很小的数）= 0.00" in lines and "0.000000001" not in row.output["清单"]
|
||||
|
||||
|
||||
def test_a_value_that_cannot_be_shown_is_said_so():
|
||||
"""有值、按口径卡的格式显示不出来（不是有限的数这类，卡上 rendered 记成「—」）：照实说，不把原值拼出来。"""
|
||||
from app.engine.nodes.metrics import _listed
|
||||
|
||||
assert _listed({"value": float("inf"), "rendered": "—"}) == "—（有值，但按设定的小数位和格式显示不出来）"
|
||||
assert _listed({"value": None, "rendered": "—", "undefined": True}).startswith("—（算不出来")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 画布校验：「分母为 0 时给 0」
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _zero_warnings(expression: str) -> list:
|
||||
g = graph(metrics=[{"id": "x", "name": "x", "expression": expression}])
|
||||
issues = validate_graph(GraphSpec.model_validate(g)).issues
|
||||
return [i for i in issues if i.code == "metrics.zero_for_undefined"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("expression", [
|
||||
"round((vars.kpi.cur - vars.kpi.prev) / vars.kpi.prev, 4) if vars.kpi.prev else 0",
|
||||
"vars.kpi.repeat / vars.kpi.visits if vars.kpi.visits else 0.0",
|
||||
"0 if not vars.kpi.visits else vars.kpi.repeat / vars.kpi.visits", # 两边对调
|
||||
"round(vars.kpi.repeat / vars.kpi.visits if vars.kpi.visits > 0 else 0, 4)", # 套在 round 里
|
||||
"vars.kpi.gmv if vars.kpi.cur else (vars.kpi.repeat / vars.kpi.visits if vars.kpi.visits else 0)", # 嵌套
|
||||
])
|
||||
def test_zero_for_undefined_is_warned(expression):
|
||||
[issue] = _zero_warnings(expression)
|
||||
assert issue.level == "warning" and issue.field == "metrics[0].expression"
|
||||
assert "None" in issue.message and "持平" in issue.message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("expression", [
|
||||
"round((vars.kpi.cur - vars.kpi.prev) / vars.kpi.prev, 4) if vars.kpi.prev else None",
|
||||
"vars.kpi.cur if vars.kpi.prev else 0", # 没有除法：0 可能就是想要的值
|
||||
"vars.kpi.repeat / vars.kpi.visits",
|
||||
"vars.kpi.repeat / vars.kpi.visits if vars.kpi.visits else 1",
|
||||
])
|
||||
def test_other_expressions_are_left_alone(expression):
|
||||
assert _zero_warnings(expression) == []
|
||||
|
||||
|
||||
def test_the_copilot_is_told_to_write_none():
|
||||
from app.api.copilot import NODE_REFERENCE
|
||||
|
||||
start = NODE_REFERENCE.index("- metrics：口径卡")
|
||||
section = NODE_REFERENCE[start:NODE_REFERENCE.index("\n- dual_check：", start)]
|
||||
assert "if vars.kpi.gmv_prev else None" in section and "不要写 else 0" in section
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 写作目录、引用、对比的说法
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _catalog(undefined: bool):
|
||||
from app.engine.evidence import build_catalog
|
||||
|
||||
card = {"kind": "metric_set", "caliber": "周报口径", "caliber_version": "v1", "artifact": "c" * 64, "metrics": [
|
||||
{"id": "wow", "name": "周环比", "unit": "", "value": None, "decimals": 4, "format": "percent_of_ratio",
|
||||
"status": "missing_input", **({"undefined": True} if undefined else {}), "expression": "…"}]}
|
||||
ledger = [{"kind": "metric_set", "node_id": "card", "exec": 1, "artifact": "c" * 64, "caliber": "周报口径",
|
||||
"version": "v1", "metrics": ["wow"]}]
|
||||
return build_catalog(nodes={"card": card}, ledger=ledger, inputs={})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("undefined, said", [(True, "算不出来"), (False, "缺少输入")])
|
||||
def test_the_writer_and_the_reason_say_which(undefined, said):
|
||||
from app.engine.evidence import compose_doc, writer_catalog
|
||||
|
||||
catalog = _catalog(undefined)
|
||||
assert catalog["m:wow"].get("undefined", False) is undefined
|
||||
text = writer_catalog(catalog).text
|
||||
line = next(ln for ln in text.splitlines() if "[[m:wow]]" in ln)
|
||||
assert ("算不出来" in line) is undefined and "不要引用" in line
|
||||
doc = compose_doc("周环比 [[m:wow]]。", catalog)
|
||||
[violation] = [v for v in doc["violations"] if v["code"] == "unresolved_ref"]
|
||||
assert said in violation["message"], violation
|
||||
|
||||
|
||||
def test_compare_says_which():
|
||||
from app.engine import compare
|
||||
|
||||
assert compare.missing_reason("这次运行", {"status": "missing_input", "undefined": True}) \
|
||||
== "这次运行这个指标算不出来（这组输入下计算式没有值，比如分母为 0）"
|
||||
assert compare.missing_reason("基准运行", {"status": "missing_input"}) == "基准运行这个指标缺少输入，没有值"
|
||||
assert compare.missing_reason("基准运行", {"status": "ok"}) is None
|
||||
@@ -0,0 +1,288 @@
|
||||
"""查询区间里这张表没有数据时，查询结果里照实说（data/coverage.py，接在 tools/datasource 的查询工具上）。
|
||||
|
||||
部署上撞过：一张到访事件表只存最新一天（每天整表重灌），周报按运行当天现算的区间取上周、上上周的到访人数，6 列全是 0，
|
||||
报告照写「上周到访人数为 0 人」。剖析记过日期范围，但对滚动表不可靠，所以运行时现查：
|
||||
|
||||
- 结果里有整列是 0 或空的数值列时，把 SQL 收窄到这一列，认出它用到的表和时间谓词；界是字面量的直接解析，按运行当天现算
|
||||
的在同一个库里算出来；这一列在表里的范围现查 MIN / MAX，查不成退回剖析记下的范围（照实标「按 X 日的剖析」）；
|
||||
- 区间和范围完全不重叠才算没有数据：结果和快照里多一项 no_data（给人和模型看的 message），运行事件里一条 query_no_data；
|
||||
没查出问题不加这个键，快照和以前一字不差；
|
||||
- 有数的列、一行都没有的结果、遮罩列、写法对不上（文本列 '20261012' 对文本界 '2026-10-12'）、认不出的一律不判。
|
||||
|
||||
库是现造的 SQLite 文件，表名、列名、数字都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.data import coverage, introspect
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.tools.datasource import build_datasource_tools
|
||||
from app.tools.registry import ToolContext, Versions
|
||||
|
||||
#: 这张表只存一天（2020-01-01），按今天现算的上周一定落在它外面
|
||||
ONLY_DAY = "20200101"
|
||||
#: 按今天现算的「上周」（SQLite 的写法，和列一样写成 'YYYYMMDD'）
|
||||
WINDOW = "{c} >= strftime('%Y%m%d', 'now', '-14 days') AND {c} < strftime('%Y%m%d', 'now', '-7 days')"
|
||||
KPI_SQL = ("SELECT (SELECT COUNT(DISTINCT v.member_id) FROM gate_events v WHERE " + WINDOW.format(c="v.event_date")
|
||||
+ ") AS 上周到访人数, (SELECT COUNT(*) FROM gate_events v WHERE " + WINDOW.format(c="v.event_date")
|
||||
+ ") AS 上周到访人次, (SELECT SUM(pv) FROM daily_pv WHERE day >= date('now', '-14 days')) AS 客流PV")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def plant(tmp_path):
|
||||
"""gate_events 只有 ONLY_DAY 一天；daily_pv 覆盖最近 30 天、每天 100。"""
|
||||
path = tmp_path / f"{uuid.uuid4().hex[:8]}.db"
|
||||
db = sqlite3.connect(path)
|
||||
db.executescript(
|
||||
"CREATE TABLE gate_events (event_date TEXT, member_id INTEGER, gate TEXT);"
|
||||
f"INSERT INTO gate_events VALUES ('{ONLY_DAY}', 1, 'A'), ('{ONLY_DAY}', 2, 'B');"
|
||||
"CREATE TABLE daily_pv (day TEXT, pv INTEGER);"
|
||||
"CREATE TABLE iso_events (paid_on TEXT, amount INTEGER);"
|
||||
"INSERT INTO iso_events VALUES ('2020-01-01', 5);")
|
||||
for back in range(30):
|
||||
db.execute("INSERT INTO daily_pv VALUES (date('now', ?), 100)", (f"-{back} days",))
|
||||
db.commit()
|
||||
db.close()
|
||||
name = f"plant_{uuid.uuid4().hex[:8]}"
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True)
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
row.schema_cache = await introspect.introspect(row)
|
||||
await session.commit()
|
||||
source_id = row.id
|
||||
yield name, source_id
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
async def _query(name: str, sql: str, events: list | None = None) -> dict:
|
||||
async def emit(event_type, **data):
|
||||
if events is not None:
|
||||
events.append((str(event_type), data))
|
||||
|
||||
async with SessionLocal() as session:
|
||||
built = await build_datasource_tools([f"db_query__{name}"], ToolContext(
|
||||
data_versions=Versions.CURRENT, run_id="run-coverage", node_id="q_kpi", emit=emit), session)
|
||||
got = await built[0].ainvoke({"sql": sql})
|
||||
return json.loads(got)
|
||||
|
||||
|
||||
async def test_zero_columns_over_a_window_without_data_are_flagged(plant):
|
||||
name, _ = plant
|
||||
events: list = []
|
||||
result = await _query(name, KPI_SQL, events)
|
||||
assert result["rows"][0][:2] == [0, 0]
|
||||
[gap] = result["no_data"]
|
||||
assert gap["columns"] == ["上周到访人数", "上周到访人次"] and gap["table"] == "gate_events"
|
||||
assert gap["column"] == "event_date" and gap["data_range"] == ["2020-01-01", "2020-01-01"] and gap["basis"] == "live"
|
||||
start, end = gap["window"]
|
||||
assert start < end and start > "2020-01-01"
|
||||
assert gap["message"].startswith("「上周到访人数」等 2 列查的是 ") and "只有 2020-01-01 这一天的数据" in gap["message"]
|
||||
assert "这些 0 不代表这段时间真的是 0，是这段时间表里没有数据" in gap["message"]
|
||||
# 有数的列（客流 PV）不受影响
|
||||
assert all("客流PV" not in g["columns"] for g in result["no_data"])
|
||||
# 快照里也有；运行事件里有一条 query_no_data
|
||||
snapshot = artifact_store.load(result["artifact"])
|
||||
assert snapshot["no_data"] == result["no_data"]
|
||||
[(etype, data)] = [e for e in events if e[1].get("code") == "query_no_data"]
|
||||
assert data["level"] == "warn" and data["no_data"] == result["no_data"]
|
||||
|
||||
|
||||
async def test_literal_windows_are_parsed_directly(plant):
|
||||
name, _ = plant
|
||||
sql = ("SELECT COUNT(*) AS 到访人次 FROM gate_events WHERE event_date >= '20261012' AND event_date < '20261019'")
|
||||
[gap] = (await _query(name, sql))["no_data"]
|
||||
assert gap["window"] == ["2026-10-12", "2026-10-18"] # < 不带时刻：截到前一天
|
||||
|
||||
|
||||
async def test_nothing_is_added_when_the_window_has_data_or_the_column_is_not_zero(plant):
|
||||
name, _ = plant
|
||||
for sql in (
|
||||
f"SELECT COUNT(*) AS n FROM gate_events WHERE event_date = '{ONLY_DAY}'", # 有数
|
||||
"SELECT SUM(pv) AS pv FROM daily_pv WHERE day >= date('now', '-7 days')", # 有数
|
||||
"SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20200101'", # 区间覆盖了那一天
|
||||
"SELECT gate, COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012' GROUP BY gate", # 一行都没有
|
||||
):
|
||||
result = await _query(name, sql)
|
||||
assert "no_data" not in result, (sql, result.get("no_data"))
|
||||
assert "no_data" not in artifact_store.load(result["artifact"]), "没查出问题：快照和以前一字不差"
|
||||
|
||||
|
||||
async def test_a_text_column_against_a_differently_written_text_bound_is_not_judged(plant):
|
||||
"""列里存的是 '20200101'，界写的是 '2026-10-12'：库按字符串比，和按日期比对不上，不判。"""
|
||||
name, _ = plant
|
||||
result = await _query(name, "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '2026-10-12'")
|
||||
assert "no_data" not in result
|
||||
# ISO 写法对 ISO 写法：照常判
|
||||
result = await _query(name, "SELECT COUNT(*) AS n FROM iso_events WHERE paid_on >= date('now', '-7 days')")
|
||||
[gap] = result["no_data"]
|
||||
assert gap["column"] == "paid_on" and gap["data_range"] == ["2020-01-01", "2020-01-01"]
|
||||
|
||||
|
||||
async def test_masked_columns_are_not_probed(plant):
|
||||
name, source_id = plant
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, source_id)
|
||||
row.options = {**(row.options or {}), "mask_columns": ["event_date"]}
|
||||
await session.commit()
|
||||
result = await _query(name, "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012'")
|
||||
assert "no_data" not in result
|
||||
|
||||
|
||||
async def test_a_failed_probe_falls_back_to_the_profile(plant, monkeypatch):
|
||||
"""现查不成（超时、出错）：退回剖析记下的范围，照实标「按 X 日的剖析」。"""
|
||||
name, _ = plant
|
||||
real = coverage._probe
|
||||
|
||||
async def probe(source, sql):
|
||||
return None if "MIN(" in sql else await real(source, sql)
|
||||
|
||||
async def facts(table):
|
||||
assert table == "gate_events"
|
||||
return {"at": "2026-10-01T03:00:00+00:00", "date_ranges": [{"column": "event_date", "min": "20260930",
|
||||
"max": "20260930"}]}
|
||||
|
||||
monkeypatch.setattr(coverage, "_probe", probe)
|
||||
async with SessionLocal() as session:
|
||||
source = await session.get(DataSource, (await _source_id(name)))
|
||||
payload = {"columns": ["n"], "rows": [[0]], "row_count": 1, "column_types": {"n": "number"}}
|
||||
[gap] = await coverage.result_gaps(source, "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012'",
|
||||
payload, facts=facts)
|
||||
assert gap["basis"] == "profile" and gap["basis_at"] == "2026-10-01"
|
||||
assert gap["data_range"] == ["2026-09-30", "2026-09-30"] and "（现查表的范围没有成功，按 2026-10-01 的剖析）" in gap["message"]
|
||||
# 剖析也没有：不判
|
||||
assert await coverage.result_gaps(source, "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012'",
|
||||
payload, facts=None) == []
|
||||
|
||||
|
||||
async def _source_id(name: str) -> str:
|
||||
from sqlalchemy import select
|
||||
|
||||
async with SessionLocal() as session:
|
||||
return (await session.execute(select(DataSource.id).where(DataSource.name == name))).scalar_one()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, kind, day, shape", [
|
||||
("2026-10-12", None, "2026-10-12", "iso"),
|
||||
("2026-10-12T08:30:00", "datetime", "2026-10-12", "date"),
|
||||
("20261012", None, "2026-10-12", "compact"),
|
||||
(20261012, None, "2026-10-12", "number"),
|
||||
("2026-13-01", None, None, None),
|
||||
("2026/10/12", None, None, None),
|
||||
("上周", None, None, None),
|
||||
])
|
||||
def test_dates_are_normalised(value, kind, day, shape):
|
||||
got = coverage._as_day(value, kind)
|
||||
assert (got.day.isoformat() if got else None) == day and (got.shape if got else None) == shape
|
||||
|
||||
|
||||
def test_zero_columns():
|
||||
payload = {"columns": ["a", "b", "c", "d"], "column_types": {"a": "number", "b": "number", "d": "text"},
|
||||
"rows": [[0, 1, None, "0"], ["0.00", 0, None, "0"]]}
|
||||
assert coverage.zero_columns(payload) == ["a", "c"]
|
||||
assert coverage.zero_columns({**payload, "rows": []}) == []
|
||||
|
||||
|
||||
async def test_the_weekly_report_shape_on_mysql(monkeypatch):
|
||||
"""部署上那张周报的形状（MySQL）：一条查询用标量子查询一次取几列，上周、上上周按运行当天现算；到访表只有最新一天，
|
||||
列存成 VARCHAR 'YYYYMMDD'。界由库算出来是 DATE 类型：MySQL 把文本列转成日期再比，照常判。两段区间各记一条。"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
last = ("event_date >= DATE_SUB(CURRENT_DATE, INTERVAL (WEEKDAY(CURRENT_DATE) + 7) DAY) "
|
||||
"AND event_date < DATE_SUB(CURRENT_DATE, INTERVAL WEEKDAY(CURRENT_DATE) DAY)")
|
||||
before = ("event_date >= DATE_SUB(CURRENT_DATE, INTERVAL (WEEKDAY(CURRENT_DATE) + 14) DAY) "
|
||||
"AND event_date < DATE_SUB(CURRENT_DATE, INTERVAL (WEEKDAY(CURRENT_DATE) + 7) DAY)")
|
||||
sql = (f"SELECT (SELECT SUM(pv) FROM pv_daily WHERE stat_day >= '2026-10-01') AS `客流PV`, "
|
||||
f"(SELECT COUNT(DISTINCT member_id) FROM gate_events WHERE {last}) AS `上周到访人数`, "
|
||||
f"(SELECT COUNT(*) FROM gate_events WHERE {last}) AS `上周到访人次`, "
|
||||
f"(SELECT COUNT(DISTINCT member_id) FROM gate_events WHERE {before}) AS `上上周到访人数`")
|
||||
days = {"(WEEKDAY(CURRENT_DATE) + 14)": "2026-09-28", "(WEEKDAY(CURRENT_DATE) + 7)": "2026-10-05",
|
||||
"(WEEKDAY(CURRENT_DATE)) DAY": "2026-10-12"} # sqlglot 写回 MySQL 时给这一项加了括号
|
||||
seen: list[str] = []
|
||||
|
||||
async def probe(source, text):
|
||||
seen.append(text)
|
||||
if "MIN(" in text:
|
||||
return ["20261019", "20261019"], ["text", "text"]
|
||||
values = []
|
||||
for part in text[len("SELECT "):].split(" AS b")[:-1]:
|
||||
values.append(next(v for k, v in days.items() if k in part))
|
||||
return values, ["date"] * len(values)
|
||||
|
||||
monkeypatch.setattr(coverage, "_probe", probe)
|
||||
source = SimpleNamespace(name="plant", kind="mysql", schema_cache=None, id=None)
|
||||
payload = {"columns": ["客流PV", "上周到访人数", "上周到访人次", "上上周到访人数"],
|
||||
"column_types": {"客流PV": "number", "上周到访人数": "number", "上周到访人次": "number",
|
||||
"上上周到访人数": "number"},
|
||||
"rows": [[3200, 0, 0, 0]], "row_count": 1}
|
||||
gaps = await coverage.result_gaps(source, sql, payload)
|
||||
by_window = {tuple(g["window"]): g for g in gaps}
|
||||
assert set(by_window) == {("2026-10-05", "2026-10-11"), ("2026-09-28", "2026-10-04")}
|
||||
assert by_window[("2026-10-05", "2026-10-11")]["columns"] == ["上周到访人数", "上周到访人次"]
|
||||
assert by_window[("2026-09-28", "2026-10-04")]["columns"] == ["上上周到访人数"]
|
||||
assert all(g["data_range"] == ["2026-10-19", "2026-10-19"] for g in gaps)
|
||||
# 表的范围同一张表、同一列只查一次；按运行当天现算的界一条 SELECT 算出来
|
||||
assert sum("MIN(" in t for t in seen) == 1
|
||||
assert all(t.startswith("SELECT ") for t in seen)
|
||||
|
||||
|
||||
async def test_a_live_range_is_kept_for_a_while(monkeypatch):
|
||||
"""现查到的范围在进程里记几分钟：一张图里几个取数节点、助手自查几轮查同一张表，不再每次全表 MIN / MAX。"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
seen: list[str] = []
|
||||
|
||||
async def probe(source, text):
|
||||
seen.append(text)
|
||||
return ["20200101", "20200101"], ["text", "text"]
|
||||
|
||||
monkeypatch.setattr(coverage, "_probe", probe)
|
||||
monkeypatch.setattr(coverage, "_LIVE", {})
|
||||
source = SimpleNamespace(name=f"plant_{uuid.uuid4().hex[:6]}", kind="sqlite", schema_cache=None, id="s1")
|
||||
payload = {"columns": ["n"], "rows": [[0]], "row_count": 1, "column_types": {"n": "number"}}
|
||||
sql = "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012'"
|
||||
for _ in range(2):
|
||||
assert len(await coverage.result_gaps(source, sql, payload)) == 1
|
||||
assert sum("MIN(" in t for t in seen) == 1
|
||||
monkeypatch.setattr(coverage, "LIVE_TTL_S", 0.0)
|
||||
coverage._LIVE.clear()
|
||||
await coverage.result_gaps(source, sql, payload)
|
||||
await coverage.result_gaps(source, sql, payload)
|
||||
assert sum("MIN(" in t for t in seen) == 3, "过期了就重新现查"
|
||||
|
||||
|
||||
async def test_a_failed_probe_is_not_retried_for_a_while(monkeypatch):
|
||||
"""现查没成功（大表全表 MIN / MAX 超时）也记几分钟：期间同一张表直接退回剖析，不再每次白等探查的时限。"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
seen: list[str] = []
|
||||
|
||||
async def probe(source, text):
|
||||
seen.append(text)
|
||||
return None
|
||||
|
||||
async def facts(table):
|
||||
return {"at": "2026-10-01T03:00:00+00:00", "date_ranges": [{"column": "event_date", "min": "20260930",
|
||||
"max": "20260930"}]}
|
||||
|
||||
monkeypatch.setattr(coverage, "_probe", probe)
|
||||
monkeypatch.setattr(coverage, "_LIVE", {})
|
||||
source = SimpleNamespace(name=f"plant_{uuid.uuid4().hex[:6]}", kind="sqlite", schema_cache=None, id="s-failed")
|
||||
payload = {"columns": ["n"], "rows": [[0]], "row_count": 1, "column_types": {"n": "number"}}
|
||||
sql = "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012'"
|
||||
for _ in range(3):
|
||||
[gap] = await coverage.result_gaps(source, sql, payload, facts=facts)
|
||||
assert gap["basis"] == "profile"
|
||||
assert sum("MIN(" in t for t in seen) == 1, "失败的现查也记着，期间不再白等"
|
||||
monkeypatch.setattr(coverage, "LIVE_TTL_S", 0.0)
|
||||
coverage._LIVE.clear()
|
||||
await coverage.result_gaps(source, sql, payload, facts=facts)
|
||||
assert sum("MIN(" in t for t in seen) == 2, "过期了就重新现查"
|
||||
@@ -350,6 +350,13 @@ export const EVIDENCE_TEXT = {
|
||||
unmatchedPlain: '无法追溯的数字：',
|
||||
uncited: '该数字未使用引用标记，系统无法核对其来源',
|
||||
missingValue: '缺少输入：口径卡本次未计算出该指标的值',
|
||||
/** 指标没有值、但不是缺输入：输入都齐，计算式自己算不出来（比如分母为 0） */
|
||||
undefinedValue: '算不出来：这组输入下计算式没有值（比如分母为 0），不是缺少输入',
|
||||
/** 指标的输入取自「查询区间里这张表没有数据」的列：这个数不代表真的是 0，出具按缺口降档 */
|
||||
noData: '区间内无数据',
|
||||
noDataFallback: '所依据的查询区间里这张表没有数据，这个数不代表真的是 0',
|
||||
/** 查询步骤：这次查询里查询区间内表里没有数据的那几列 */
|
||||
noDataTitle: '查询区间内表里没有数据',
|
||||
/** 指标拿截断的查询结果整组算出来（计数、求和……）：值只算到了取回的那部分，出具按缺口降档 */
|
||||
incomplete: '结果不完整',
|
||||
incompleteFallback: '基于被截断的查询结果计算，结果不完整',
|
||||
|
||||
@@ -1361,6 +1361,8 @@ function MetricPart({ step, entry, inputs, runId, runClass, seal, pending, faile
|
||||
// 口径卡里 rendered 为「—」有两种：value 为 null 是缺输入，不为 null 是有值但按格式显示不出来
|
||||
const dash = rendered === NONE || entry?.rendered === NONE
|
||||
const missing = entry?.status === 'missing_input' || (value == null && (dash || !resolved))
|
||||
// 没有值的两种：缺输入，和输入都齐、计算式自己算不出来（分母为 0，口径卡记了 undefined）
|
||||
const undefinedValue = missing && (step?.undefined === true || entry?.undefined === true)
|
||||
const unshowable = !missing && dash && value != null
|
||||
// 拿截断的查询结果整组算出来的：值是真的算出来的，但只算到了取回的那部分，出具已按缺口降档
|
||||
const incomplete = step?.incomplete === true || entry?.incomplete === true
|
||||
@@ -1368,6 +1370,9 @@ function MetricPart({ step, entry, inputs, runId, runClass, seal, pending, faile
|
||||
// 所依据的查询对照数据目录查出了错误级的问题：值照算，结果不可靠，出具同样按缺口降档。问题和改法在查询步骤里
|
||||
const sqlFailed = step?.sql_check_failed === true || entry?.sql_check_failed === true
|
||||
const sqlReason = step?.sql_check_reason || entry?.sql_check_reason || EVIDENCE_TEXT.sqlCheckFallback
|
||||
// 输入取自「查询区间里这张表没有数据」的列：值照算，那个 0 不代表真的是 0，出具同样按缺口降档
|
||||
const noData = step?.no_data === true || entry?.no_data === true
|
||||
const noDataReason = step?.no_data_reason || entry?.no_data_reason || EVIDENCE_TEXT.noDataFallback
|
||||
// 期 4：双盲复算复核过这个指标才有
|
||||
const dualCheck = dualCheckOf(step)
|
||||
// 输入追没追到原始数据：和「有出处」（确定性复算一致）分开说。卡上没有输入、老服务端没给时不说
|
||||
@@ -1394,6 +1399,12 @@ function MetricPart({ step, entry, inputs, runId, runClass, seal, pending, faile
|
||||
{EVIDENCE_TEXT.sqlCheckFailed}
|
||||
</span>
|
||||
)}
|
||||
{noData && (
|
||||
<span className="chip" data-ev-no-data="" title={noDataReason}
|
||||
style={{ color: 'var(--st-waiting)', borderColor: 'var(--st-waiting)' }}>
|
||||
{EVIDENCE_TEXT.noData}
|
||||
</span>
|
||||
)}
|
||||
{!!untraced?.length && (
|
||||
<span className="chip" data-ev-untraced-chip="" title={EVIDENCE_TEXT.originChipHint}
|
||||
style={{ color: 'var(--st-waiting)', borderColor: 'var(--st-waiting)' }}>
|
||||
@@ -1416,13 +1427,21 @@ function MetricPart({ step, entry, inputs, runId, runClass, seal, pending, faile
|
||||
)}
|
||||
{dualCheck && <DualCheckLine check={dualCheck} formal={runClass === 'formal'} />}
|
||||
<Integrity step={step} seal={seal} />
|
||||
{missing && <p className="mt-1" style={{ color: 'var(--st-waiting)' }} data-ev-missing="">{EVIDENCE_TEXT.missingValue}</p>}
|
||||
{missing && (
|
||||
<p className="mt-1" style={{ color: 'var(--st-waiting)' }} data-ev-missing=""
|
||||
{...(undefinedValue ? { 'data-ev-undefined': '' } : {})}>
|
||||
{undefinedValue ? EVIDENCE_TEXT.undefinedValue : EVIDENCE_TEXT.missingValue}
|
||||
</p>
|
||||
)}
|
||||
{incomplete && (
|
||||
<p className="mt-1" style={{ color: 'var(--st-waiting)' }} data-ev-incomplete-reason="">{incompleteReason}</p>
|
||||
)}
|
||||
{sqlFailed && (
|
||||
<p className="mt-1" style={{ color: 'var(--st-waiting)' }} data-ev-sql-check-reason="">{sqlReason}</p>
|
||||
)}
|
||||
{noData && (
|
||||
<p className="mt-1" style={{ color: 'var(--st-waiting)' }} data-ev-no-data-reason="">{noDataReason}</p>
|
||||
)}
|
||||
{unshowable && (
|
||||
<p className="mt-1" style={{ color: 'var(--st-waiting)' }} data-ev-unshowable="">
|
||||
{EVIDENCE_TEXT.unshowable(valueText(value))}
|
||||
@@ -1657,6 +1676,10 @@ function UpstreamPart({ id, step, seal, linkOf }: {
|
||||
<span className="chip" title={step.sql_check_reason || EVIDENCE_TEXT.sqlCheckFallback}
|
||||
style={{ color: 'var(--st-waiting)', borderColor: 'var(--st-waiting)' }}>{EVIDENCE_TEXT.sqlCheckFailed}</span>
|
||||
)}
|
||||
{step.no_data === true && (
|
||||
<span className="chip" title={step.no_data_reason || EVIDENCE_TEXT.noDataFallback}
|
||||
style={{ color: 'var(--st-waiting)', borderColor: 'var(--st-waiting)' }}>{EVIDENCE_TEXT.noData}</span>
|
||||
)}
|
||||
{host.onNode && <NodeChip id={step.node_id} />}
|
||||
</div>
|
||||
<Integrity step={step} seal={seal} />
|
||||
@@ -1854,6 +1877,20 @@ function QueryPart({ id, step, seal, masked, pick }: {
|
||||
<SqlCheckList checks={step.checks} />
|
||||
</div>
|
||||
)}
|
||||
{Array.isArray(step.no_data) && step.no_data.length > 0 && (
|
||||
// 查询区间里这张表没有数据的那几列（查询时现查）：这些 0 不代表真的是 0。没有问题时接口不给这个键
|
||||
<div className="mb-1.5 text-2xs" data-ev-no-data-list="">
|
||||
<div className="mb-0.5 text-faint">{EVIDENCE_TEXT.noDataTitle}</div>
|
||||
<ul className="space-y-0.5">
|
||||
{step.no_data.map((g, k) => (
|
||||
<li key={k} className="flex items-start gap-1 leading-snug" style={{ color: 'var(--st-waiting)' }}>
|
||||
<AlertTriangle size={10} className="mt-[2px] shrink-0" aria-hidden />
|
||||
<span>{g.message}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
{!!step.merge?.warnings.length && (
|
||||
<div className="mb-1.5 text-2xs" data-ev-merge-warnings="">
|
||||
<div className="mb-0.5 text-faint">{MERGE_TEXT.executionWarnings}</div>
|
||||
|
||||
@@ -2346,6 +2346,21 @@ export interface EvidenceMerge {
|
||||
}
|
||||
|
||||
/** 指标的一个输入追不到原始数据：出处追到哪个节点为止。认不出是哪个节点时 node_id、label 为 null */
|
||||
/** 查询区间里这张表没有数据的那几列（服务端 data/coverage.result_gaps） */
|
||||
export interface EvidenceNoData {
|
||||
columns: string[]
|
||||
table?: string
|
||||
column?: string
|
||||
/** 查的区间（按天，两头都含），没有限制的那一头为 null */
|
||||
window?: (string | null)[]
|
||||
/** 表里这一列的范围；表里一个值都没有时为 null */
|
||||
data_range?: (string | null)[] | null
|
||||
/** live 现查；profile / observed 现查不成，按剖析、取值观察 */
|
||||
basis?: string
|
||||
basis_at?: string
|
||||
message: string
|
||||
}
|
||||
|
||||
export interface EvidenceUntraced {
|
||||
path: string
|
||||
node_id: string | null
|
||||
@@ -2399,6 +2414,15 @@ export interface EvidenceStep extends Omit<EvidenceInput, 'status'> {
|
||||
sql_check_failed?: boolean
|
||||
/** metric：没通过的原因（人话，「所依据的查询未通过 SQL 检查（…），结果不可靠」） */
|
||||
sql_check_reason?: string
|
||||
/** metric：没有值，不是缺输入，是输入都齐、计算式自己算不出来（分母为 0）。状态仍是 missing_input */
|
||||
undefined?: boolean
|
||||
/**
|
||||
* metric：输入取自「查询区间里这张表没有数据」的列，值照算，出具按缺口降档（true）；
|
||||
* query：这次查询里这样的几列（服务端 data/coverage.py，查询时记进快照）。没有问题时没有这个键
|
||||
*/
|
||||
no_data?: boolean | EvidenceNoData[]
|
||||
/** metric：没有数据的原因（人话，「所依据的查询区间里这张表没有数据：…」） */
|
||||
no_data_reason?: string
|
||||
/** metric（期 4）：这个指标的双盲复算状态。没有双盲复算复核过这份指标集时没有这个键 */
|
||||
dual_check?: EvidenceDualCheck
|
||||
/**
|
||||
|
||||
+49
-1
@@ -149,6 +149,23 @@ def _ddl(t: Target) -> list[str]:
|
||||
return out
|
||||
|
||||
|
||||
def _snapshot(t: Target) -> list[str]:
|
||||
"""覆盖检查（data/coverage.py）的夹具：只存当天的滚动表（每天整表重灌的那种）。日期按库自己的当前日期写，避开容器
|
||||
和本机时区不一致；daily_snapshot 的日期按文字 'YYYYMMDD' 存（部署上撞过的写法，只在 MySQL / MariaDB 上判），
|
||||
daily_snapshot_d 按 DATE 存。"""
|
||||
vc = "VARCHAR2" if t.oracle else "VARCHAR"
|
||||
today_text = ("TO_CHAR(SYSDATE, 'YYYYMMDD')" if t.oracle
|
||||
else "REPLACE(CAST(CURRENT_DATE AS CHAR), '-', '')" if t.mysql
|
||||
else "REPLACE(CAST(CURRENT_DATE AS VARCHAR), '-', '')")
|
||||
today = "TRUNC(SYSDATE)" if t.oracle else "CURRENT_DATE"
|
||||
out = [f"CREATE TABLE daily_snapshot (event_day {vc}(8), visitor {vc}(16))",
|
||||
f"CREATE TABLE daily_snapshot_d (event_day DATE, visitor {vc}(16))"]
|
||||
for who in ("v1", "v2", "v3"):
|
||||
out.append(f"INSERT INTO daily_snapshot (event_day, visitor) VALUES ({today_text}, '{who}')")
|
||||
out.append(f"INSERT INTO daily_snapshot_d (event_day, visitor) VALUES ({today}, '{who}')")
|
||||
return out
|
||||
|
||||
|
||||
def _paid(t: Target, o: dict[str, Any]) -> str:
|
||||
if o["paid_at"] is not None:
|
||||
stamp = o["paid_at"].strftime("%Y-%m-%d %H:%M:%S")
|
||||
@@ -212,7 +229,7 @@ async def _setup(t: Target) -> None:
|
||||
if t.mysql:
|
||||
# 关掉严格模式才写得进零日期（老库里常见的情形）
|
||||
await conn.execute(text("SET SESSION sql_mode = ''"))
|
||||
for sql in _ddl(t) + _rows(t) + _events(1, N_EVENTS_OLD):
|
||||
for sql in _ddl(t) + _rows(t) + _events(1, N_EVENTS_OLD) + _snapshot(t):
|
||||
await conn.exec_driver_sql(sql)
|
||||
async with engine.begin() as conn:
|
||||
for sql in _analyze(t, "events") + _analyze(t, "orders") + _analyze(t, "stores"):
|
||||
@@ -421,10 +438,41 @@ async def run_target(t: Target) -> None:
|
||||
res = await data_engine.run_query(source, d.max_date_sql(tbl, paid_col))
|
||||
check(bool(res.rows) and str(res.rows[0][0])[:19].replace("T", " ") == str(EXPECT["paid_at"][1]),
|
||||
f"max_date_sql 直接跑：{res.rows}")
|
||||
await _coverage(t, source)
|
||||
finally:
|
||||
await data_engine.engines.invalidate(source.id)
|
||||
|
||||
|
||||
async def _coverage(t: Target, source: Any) -> None:
|
||||
"""覆盖检查（data/coverage.py）在真库上：按运行当天现算的上周区间，表里只有当天的数据 → 判成没有数据；区间包含当天、
|
||||
只是别的条件筛空的 → 不报。界在库里算（各方言自己的写法），表的范围现查 MIN / MAX。"""
|
||||
from app.data import coverage
|
||||
|
||||
last_week = {"mysql": ("event_day >= DATE_SUB(CURRENT_DATE, INTERVAL (WEEKDAY(CURRENT_DATE) + 7) DAY) "
|
||||
"AND event_day < DATE_SUB(CURRENT_DATE, INTERVAL WEEKDAY(CURRENT_DATE) DAY)"),
|
||||
"pg": ("event_day >= date_trunc('week', CURRENT_DATE) - INTERVAL '7 days' "
|
||||
"AND event_day < date_trunc('week', CURRENT_DATE)"),
|
||||
"oracle": "event_day >= TRUNC(SYSDATE, 'IW') - 7 AND event_day < TRUNC(SYSDATE, 'IW')"}
|
||||
recent = {"mysql": "event_day >= DATE_SUB(CURRENT_DATE, INTERVAL 1 DAY)",
|
||||
"pg": "event_day >= CURRENT_DATE - INTERVAL '1 day'",
|
||||
"oracle": "event_day >= TRUNC(SYSDATE) - 1"}
|
||||
flavor = "mysql" if t.mysql else "pg" if t.pg else "oracle"
|
||||
today_sql = "SELECT TRUNC(SYSDATE) FROM DUAL" if t.oracle else "SELECT CURRENT_DATE"
|
||||
today = str((await data_engine.run_query(source, today_sql)).rows[0][0])[:10]
|
||||
for table in ["daily_snapshot_d"] + (["daily_snapshot"] if t.mysql else []):
|
||||
sql = f"SELECT COUNT(DISTINCT visitor) AS n FROM {table} WHERE {last_week[flavor]}"
|
||||
gaps = await coverage.result_gaps(source, sql, (await data_engine.run_query(source, sql)).to_payload())
|
||||
check(len(gaps) == 1 and gaps[0]["data_range"] == [today, today]
|
||||
and [c.lower() for c in gaps[0]["columns"]] == ["n"],
|
||||
f"覆盖检查：{table} 只有当天的数据，上周的区间判成没有数据：{gaps}")
|
||||
pre = await coverage.sql_gaps(source, sql)
|
||||
check(len(pre) == 1, f"助手自查：{table} 上周的区间判成没有数据：{pre}")
|
||||
fine = f"SELECT COUNT(DISTINCT visitor) AS n FROM {table} WHERE visitor = 'nobody' AND {recent[flavor]}"
|
||||
quiet = await coverage.result_gaps(source, fine, (await data_engine.run_query(source, fine)).to_payload())
|
||||
check(quiet == [], f"覆盖检查：{table} 区间里有数据、只是别的条件筛空的不报：{quiet}")
|
||||
check(await coverage.sql_gaps(source, fine) == [], f"助手自查：{table} 区间包含当天的不报")
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
only = set((os.environ.get("DIALECT_IT_ONLY") or "pg,mysql,mariadb,oracle").split(","))
|
||||
plan = [("pg", "postgresql", "DIALECT_IT_PG_PORT"), ("mysql", "mysql", "DIALECT_IT_MYSQL_PORT"),
|
||||
|
||||
Reference in New Issue
Block a user