diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 91be55cf..fb8b7ade 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -207,6 +207,9 @@ NODE_REFERENCE = """\ config: {caliber, caliber_version, metrics:[{id, name, unit, decimals, format, expression}], on_missing, assign_to} expression 是受限表达式（不是模板）：vars.kpi.gmv、cell(nodes.fetch, 0, 'gmv')、 round((vars.kpi.gmv - vars.kpi.gmv_prev) / vars.kpi.gmv_prev * 100, 1) + **分母可能为 0**（上期为 0、这段时间没有数据）时写「… if 分母 else None」： + round((vars.kpi.gmv - vars.kpi.gmv_prev) / vars.kpi.gmv_prev * 100, 1) if vars.kpi.gmv_prev else None。 + 算不出来就是空值（报告里显示「—」，写作者照实说算不出来）；不要写 else 0——0 会被读成持平、0% format：thousands（默认，千分位）/ plain（不分组，年份编号这类）/ percent_of_ratio（值是 0.0235 这样的比率，显示成 2.35%）； decimals 写 -15 到 15 的整数；percent_of_ratio 的 decimals 按比率算：要显示两位百分比（2.35%）写 4， 写 2 只剩「2%」，0.004 这样的小比率还会显示不出来、报告引用不了。on_missing 默认 fail（缺输入整张卡失败）；写 null 则缺输入的指标记为空值，交给出具契约判档 @@ -293,7 +296,9 @@ NODE_REFERENCE = """\ 6. 问题里的「今天、昨天、上周、本月、去年」这类相对日期一律按运行当天算：SQL 用数据源方言里取当天的写法（CURRENT_DATE、 CURDATE()、date('now')、SYSDATE），agent 先调 current_time 拿到当天日期再写条件（agent 的 prompt 里写明这一点、tools 绑上 current_time）。不要用表里最大的日期（MAX(日期)）去推「上周」「本月」：数据没更新到那段时间时，推出来的是另一段，报告还会 - 把它当成上周来写。想让报告说清数据最新到哪天，另取一列 MAX(日期) AS 数据最新日期 + 把它当成上周来写。想让报告说清数据最新到哪天，另取一列 MAX(日期) AS 数据最新日期。 + 写按周期取数的 SQL 前先看表的日期范围（look 结果里的剖析、取值观察）：只有一天的数据、或者最新日期早于要查的区间的表， + 不能用来算这段时间（查出来全是 0）——换有历史的表；没有这样的表，就在说明里照实告诉用户这段时间没有数据 模型字段（llm / agent / supervisor / report 的 config.model）： - **不要填**。留空表示跟随当前供应商的默认模型，这几乎总是对的。 @@ -950,6 +955,7 @@ async def generate( applier.replay_onto(prev_graph) async for _op in gen.repair(): pass + covered = await gen.checker.coverage(level="warning") if isinstance(gen.checker, GraphChecker) else [] try: spec = GraphSpec.model_validate({"nodes": list(nodes.values()), "edges": edges}) @@ -968,7 +974,7 @@ async def generate( explanation=applier.explanation or str((gen.asked or {}).get("question") or ""), issues=[i.model_dump() for i in report.issues if i.level != "info"] + _issue_dicts(scope_issues(out["nodes"], scope), "datasource_out_of_scope") - + dropped_tool_warnings(changes, payload.instruction), + + dropped_tool_warnings(changes, payload.instruction) + covered, layout=layout, tool_changes=changes, ) @@ -1809,6 +1815,9 @@ async def answer_ops(payload: GenerateIn, *, catalog_view: Any = None): # 能跑，只是跑出来的是模型「假设」查过库的答案 changes = tool_changes(baseline, list(nodes.values())) warnings = dropped_tool_warnings(changes, payload.instruction) + # 工具节点写死的 SQL 查的区间里表里没有数据（data/coverage.py）：自查时交回模型改过一轮，仍在的不挡运行，留在检查 + # 结果里当提醒（模型照要求在说明里告诉用户） + warnings += await gen.checker.coverage(level="warning") if isinstance(gen.checker, GraphChecker) else [] # 建完要不要跑，由模型在 done.run 里表态；问了用户的不跑：要等他回答 autorun = applier.autorun and gen.asked is None if remaining: diff --git a/backend/app/api/copilot_gen.py b/backend/app/api/copilot_gen.py index fbe0ce58..68c7a888 100644 --- a/backend/app/api/copilot_gen.py +++ b/backend/app/api/copilot_gen.py @@ -723,6 +723,11 @@ class GraphChecker: self.defaults = defaults self.checkers = checkers self.known = known + #: 工具节点写死的 SQL 查的区间里表里没有数据（data/coverage.py）：同一个节点、同一条 SQL 只查一次库 + self._coverage: dict[tuple[str, str], list[dict[str, Any]]] = {} + self._coverage_cache: Any = None + #: 这类问题已经交回模型改过一轮：之后不再挡（修正后仍在的留在检查结果里当提醒，由模型在说明里照实告诉用户） + self._coverage_given = False def blocking(self) -> list[dict[str, Any]] | None: from app.api.copilot import _blocking_issues @@ -735,7 +740,10 @@ class GraphChecker: errors = self.blocking() refused = _refused_issues(self.applier) if self.known is None: - return [*refused, *(errors or [])] + # 查的区间里表里没有数据：交回模型改一轮（换有这段数据的表，或者在说明里照实告诉用户）；改过一轮仍在的不挡 + covered = [] if self._coverage_given or errors is None else await self.coverage() + self._coverage_given = self._coverage_given or bool(covered) + return [*refused, *(errors or []), *covered] from app.engine.autofix import _sig if not self.applier.effective: @@ -747,6 +755,77 @@ class GraphChecker: return _repair_request(self.applier.nodes, self.applier.edges, issues, looks=looks_text) + async def coverage(self, *, level: str = "error") -> list[dict[str, Any]]: + """工具节点写死的 SQL 里，查的区间和表里的范围完全不重叠的（data/coverage.sql_gaps）：搭图时没有查询结果，SQL 里 + 所有带时间谓词的表都查（按表去重、限时，不跑原 SQL，只跑算界和现查范围的两条小查询）。一条一个问题（形状同自查的 + 问题，code 是 sql_no_data）；level 给 warning 时是收尾那一步的提醒。""" + out: list[dict[str, Any]] = [] + for nid, node in self.applier.nodes.items(): + cfg = ((node or {}).get("data") or {}).get("config") or {} + tool, args = cfg.get("tool"), cfg.get("args") + sql = args.get("sql") if isinstance(args, dict) else None + if not isinstance(tool, str) or not tool.startswith("db_query__") or not isinstance(sql, str) \ + or not sql.strip() or "{{" in sql: + continue + key = (nid, sql) + if key not in self._coverage: + self._coverage[key] = await self._gaps(tool[len("db_query__"):], sql) + out.extend({"level": level, "node_id": nid, "edge_id": None, "field": "args.sql", "code": "sql_no_data", + "message": f"{gap['message']}：这段时间表里没有数据，查出来会是 0", + "for_model": f"{gap['message']}：换有这段数据的表；没有这样的表，就在 done 的 explanation 里照实" + "告诉用户这段时间表里没有数据、数据只到哪天（写具体日期）"} + for gap in self._coverage[key]) + return out + + async def _gaps(self, name: str, sql: str) -> list[dict[str, Any]]: + from app.data import coverage + from app.data.engine import masked_columns + + row = next((s for s in self.sources or [] if getattr(s, "name", None) == name), None) + if row is None or not getattr(row, "enabled", True): + return [] + if self._coverage_cache is None: + self._coverage_cache = coverage.Cache() + try: + source = await _probe_source(row) + except Exception: # noqa: BLE001 - 上传表格没有可用的版本这类：不判 + return [] + return await coverage.sql_gaps(source, sql, facts=_profile_facts(row), masked=masked_columns(row.options), + cache=self._coverage_cache, budget=COVERAGE_BUDGET_S) + + +#: 自查时一条 SQL 的覆盖检查最多花多少秒（data/coverage.py）：搭图要等它，比查询时的 BUDGET_S 短 +COVERAGE_BUDGET_S = 8.0 + + +async def _probe_source(row: Any) -> Any: + """自查时探查用的源：上传表格换成当前版本的快照视图（table_versions.resolve_source），别的原样。""" + from app.data import table_versions + from app.db.base import SessionLocal + + if (getattr(row, "origin", None) or "manual") != "upload": + return row + async with SessionLocal() as session: + return await table_versions.resolve_source(session, row) + + +def _profile_facts(row: Any) -> Any: + """现查表的范围不成时退回的剖析事实（catalog_profile_jobs.last_facts）。""" + async def facts(table: str) -> Any: + from app.data import catalog as data_catalog + from app.data.catalog_profile_jobs import last_facts + from app.db.base import SessionLocal + + name = data_catalog.resolve_table_name(getattr(row, "schema_cache", None), table) or table + async with SessionLocal() as session: + found = (await last_facts(session, str(row.id), [name])).get(name) + current = getattr(row, "current_snapshot_id", None) if (getattr(row, "origin", None) or "") == "upload" else None + if found and current and found.get("snapshot_id") != current: + return None # 上传表格：剖析时是另一版数据 + return found + + return facts + # ========================================================================== # 应用的落地 diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index 8acb580d..1a88fc7d 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -992,6 +992,10 @@ async def _metric_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[s # 来源查询没通过 SQL 检查：面板标出来并写明原因（问题和改法在查询步骤的 checks 里）。没问题的没有这两个键 **({"sql_check_failed": True, "sql_check_reason": source.get("sql_check_reason") or ""} if source.get("sql_check_failed") else {}), + # 输入都齐、计算式自己算不出来（分母为 0）：面板说「算不出来」，不说「缺少输入」。有值的没有这个键 + **({"undefined": True} if source.get("undefined") and source.get("value") is None else {}), + # 输入取自「查询区间里这张表没有数据」的列（data/coverage.py）：面板标出来并写明原因。没问题的没有这两个键 + **({"no_data": True, "no_data_reason": source.get("no_data_reason") or ""} if source.get("no_data") else {}), "node_id": entry.get("node_id"), "artifact": artifact or None, "eid": cite.get("eid"), "eid_ok": cite.get("eid") == entry.get("eid") == make_eid("metric", artifact or None, {"metric": metric_id}), "hash_ok": hash_ok and metric is not None, @@ -1091,6 +1095,8 @@ async def _upstream_steps(metric: dict[str, Any], sealed: _Sealed, masks: "_Mask if found.get("incomplete") else {}), **({"sql_check_failed": True, "sql_check_reason": found.get("sql_check_reason") or ""} if found.get("sql_check_failed") else {}), + **({"undefined": True} if found.get("undefined") and found.get("value") is None else {}), + **({"no_data": True, "no_data_reason": found.get("no_data_reason") or ""} if found.get("no_data") else {}), "untraced": _untraced_steps(found, sealed, node), }) if withheld := {path: _held_word(h) for path, h in held.items() if h.kind != "table"}: @@ -2571,6 +2577,8 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str column_types=snap.get("column_types") if isinstance(snap.get("column_types"), dict) else {}, **({"provenance": True} if hint and snap.get("data_version") and not merged else {}), **({"checks": shown.checks(checks)} if (checks := _query_checks(snap)) else {}), + # 查询区间里这张表没有数据的那几列（data/coverage.py，查询时记进快照）。没有就不加这个键 + **({"no_data": gaps} if (gaps := _no_data_view(snap)) else {}), **_catalog_of(snap)) if shown.changed: notes.append(SQL_MASKED_NOTE) @@ -2585,6 +2593,17 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str return step +#: 查询步骤上 no_data 每一条给面板的字段（data/coverage.result_gaps 的形状） +_NO_DATA_FIELDS = ("columns", "table", "column", "window", "data_range", "basis", "basis_at", "message") + + +def _no_data_view(snap: dict[str, Any]) -> list[dict[str, Any]]: + """快照里记的「查询区间里这张表没有数据」（no_data）：只给面板要的那几个字段，认不出的条目不给。""" + items = snap.get("no_data") if isinstance(snap.get("no_data"), list) else [] + return [{k: item[k] for k in _NO_DATA_FIELDS if k in item} for item in items + if isinstance(item, dict) and isinstance(item.get("message"), str)] + + class _Shown: """查询步骤上要展示的 SQL、合并 SQL、SQL 检查的片段、合并警告：里面写着的遮罩值（secrets，这次运行查到过的， masking._Secrets）换成「已遮罩」，换了写法的长号码也认。secrets 为 None 时原样交回。changed：换过东西，查询步骤 diff --git a/backend/app/apps/resolve.py b/backend/app/apps/resolve.py index f033259f..9174f987 100644 --- a/backend/app/apps/resolve.py +++ b/backend/app/apps/resolve.py @@ -75,6 +75,24 @@ STATUS_TEXT = {"queued": "排队中", "running": "运行中", "interrupted": " REJECTED_TEXT = "被驳回" TIER_TEXT = {"formal": "完整出具", "degraded": "降档出具", "withheld": "不予出具"} NO_TIER = "未声明出具契约" +#: 成果没配出具契约（印章上是「未声明出具契约」）时，本该降档的缺口照实写进 notes：不然实例页上看不出来 +UNGATED_NO_DATA = "指标{names}所依据的查询区间里表里没有数据，这些 0 不代表真的是 0（成果没有配出具契约，没有按缺口降档）" +UNGATED_DUAL = "双盲复算有缺口：{gaps}（成果没有配出具契约，没有按缺口降档）" + + +def ungated_notes(cards: list[dict[str, Any]], dual: dict[str, Any] | None) -> list[str]: + """没配出具契约的成果，印章上要照实写的缺口：口径卡里输入取自「查询区间里这张表没有数据」的列的指标（no_data）， + 双盲复算的缺口（产出的 issuance_gaps：无法复算、审核人没采用原流程……）。配了契约的这些已经按缺口降档，不另写。""" + notes: list[str] = [] + names = list(dict.fromkeys(str(m.get("name") or m.get("id")) for card in cards for m in card.get("metrics") or [] + if isinstance(m, dict) and m.get("no_data"))) + if names: + shown = "".join(f"「{n}」" for n in names[:3]) + (f"等 {len(names)} 个" if len(names) > 3 else "") + notes.append(UNGATED_NO_DATA.format(names=shown)) + gaps = [str(g) for g in (dual or {}).get("issuance_gaps") or [] if g] + if gaps: + notes.append(UNGATED_DUAL.format(gaps="；".join(gaps[:2]) + ("等" if len(gaps) > 2 else ""))) + return notes #: 蓝图试运行（docs/design/normal-user-apps.md N4）：印章的来源、说明，趋势只有一个点时的说明，预览里入口按钮为什么点不了 TRIAL_FROM = "试运行" TRIAL_NOTE = "数据是真的，但这不是正式结果：发布之后才会按设定出数" @@ -315,7 +333,8 @@ async def stamp_of(session: AsyncSession, source: Source) -> dict[str, Any]: if run.data_versions: parts.append("数据版本 " + "、".join(_version_text(v) for v in run.data_versions.values() if isinstance(v, dict))) calibers: list[dict[str, str]] = [] - for _, _, card in _sealed_cards(sealed): + cards = [card for _, _, card in _sealed_cards(sealed)] + for card in cards: item = {"name": str(card.get("caliber") or ""), "version": str(card.get("caliber_version") or "")} if item["name"] and item not in calibers: calibers.append(item) @@ -339,6 +358,9 @@ async def stamp_of(session: AsyncSession, source: Source) -> dict[str, Any]: # 个人应用最近一次刷新没成功、这里给的是上一次的（_exploratory_source） notes.append(source.message) notes += [gap for entry in (run.data_versions or {}).values() if (gap := _records_gap(entry))] + if tier not in TIER_TEXT: + # 没配出具契约：双盲复算的缺口、查询区间里没有数据的指标不会降档，照实写在这里 + notes += ungated_notes(cards, sealed.payload_of(sealed.nodes_of("dual_check"))) if not sealed.trusted: notes.append("这次运行还没有封存（或封存核对不一致），数字仅供参考") return { diff --git a/backend/app/bundle/build.py b/backend/app/bundle/build.py index f742a610..61219418 100644 --- a/backend/app/bundle/build.py +++ b/backend/app/bundle/build.py @@ -498,9 +498,10 @@ def _snapshot_hidden(content: dict[str, Any], source_masks: Mapping[str, Any]) - # -------------------------------------------------------------------------- #: 数据源查询工具（tools/datasource.py）落的查询快照里会有的键。有别的键的（比如核查 agent 在查询时就把遮罩列换成 -#: 「已遮罩」、另记下了遮了哪些列）不知道它记着什么，照现有规则只给 id +#: 「已遮罩」、另记下了遮了哪些列）不知道它记着什么，照现有规则只给 id。no_data 是查询区间里这张表没有数据的那几列 +#: （data/coverage.py）：只有日期和表名、列名，遮罩列上的不探查也不记 _QUERY_KEYS = frozenset({"columns", "rows", "row_count", "truncated", "elapsed_ms", "sql", "column_types", "source", - "checks", "schema_artifact", "mask_columns", "data_version"}) + "checks", "schema_artifact", "mask_columns", "data_version", "no_data"}) #: 导出时只知道数据源设的遮罩、不知道是哪种库：SQL 按每种支持的库的读法各解析一遍（None 是 sqlglot 的通用读法）， #: 至少一种读得通，读得通的每一种都要过关 _READINGS = (None, "sqlite", "mysql", "postgres", "oracle") diff --git a/backend/app/chat/turn.py b/backend/app/chat/turn.py index f6796d85..1161821d 100644 --- a/backend/app/chat/turn.py +++ b/backend/app/chat/turn.py @@ -142,11 +142,16 @@ def review_reason(review: Mapping[str, Any]) -> str: def retry_after_review(review: Mapping[str, Any] | None, attempt: int) -> bool: - """复核之后要不要带着原图重来一次：复核说换个配置重跑大概率就好（retry）、答案不可信（severity=broken）、这一轮还没 - 试满（attempt 从 0 数，attempt + 1 < MAX_ATTEMPTS）。""" + """复核之后要不要带着原图重来一次：复核说换个配置重跑大概率就好（retry）、答案不可信（severity=broken）或者有值得 + 重建一次的缺口（engine/review.REBUILD：查询区间里这张表没有数据，换一张有这段数据的表大概率就好）、这一轮还没试满 + （attempt 从 0 数，attempt + 1 < MAX_ATTEMPTS）。""" + from app.engine.review import REBUILD + if not review: return False - return bool(review.get("retry")) and review.get("severity") == "broken" and attempt + 1 < MAX_ATTEMPTS + rebuild = any(isinstance(s, Mapping) and s.get("kind") in REBUILD for s in review.get("signals") or []) + return bool(review.get("retry")) and (review.get("severity") == "broken" or rebuild) \ + and attempt + 1 < MAX_ATTEMPTS def retry_instruction(question: str, reason: str, has_base: bool) -> str: diff --git a/backend/app/data/coverage.py b/backend/app/data/coverage.py new file mode 100644 index 00000000..e7db5445 --- /dev/null +++ b/backend/app/data/coverage.py @@ -0,0 +1,810 @@ +"""查询区间里这张表有没有数据：运行时现查（查询结果整列是 0、助手搭图自查时用）。 + +**为什么要有它。** 部署上撞过：一份周报按 `event_date >= DATE_SUB(CURRENT_DATE, INTERVAL (WEEKDAY(CURRENT_DATE)+7) DAY) +AND event_date < …` 取上周、上上周的到访人数，那张表却只存最新一天（每天整表重灌），6 列全是 0，口径卡算出一排 0 和 0%， +周报照写「上周到访人数为 0 人」「周环比 0.00%」。这些 0 不是「没有到访」，是这段时间表里根本没有数据。剖析记过日期列的 +最小、最大值，但对滚动表不可靠（剖析那天表里是前一天的数据，运行时已经换成当天），所以要运行时现查。 + +**做什么。** 两个入口共用一套： + +- result_gaps：一次查询的结果里有「整列都是 0 或空」的数值列时（结果一行都没有的不归这里管），对每个这样的列，把 SQL + 收窄到算出这一列的那一项（去掉别的选择项、分组、HAVING、排序、取行数，同双盲复算差异说明的「收窄到这一格」，这里只做 + 分析、不执行），取出它用到的表和时间谓词（列、比较符、界）； +- sql_gaps：助手搭图自查时，工具节点里写死的 SQL 还没跑、没有结果：对 SQL 里所有带时间谓词的表都查。 + +界是字面量的直接解析；按运行当天现算的（不含列引用的表达式：CURRENT_DATE、DATE_SUB(…)、date('now', …)……）用同一个数据源 +跑一条 `SELECT 界1, 界2 …` 在库里算出来（Oracle 加 FROM DUAL）。这一列在这张表里的实际范围现查 `SELECT MIN(列), MAX(列) +FROM 表`（每条探查 PROBE_TIMEOUT_S 秒）；超时或失败退回剖析、取值观察记下的范围（facts，照实标「按 X 日的剖析」），都没有 +就不判。日期按常见写法归一（DATE / DATETIME、'YYYY-MM-DD'、'YYYYMMDD'、8 位整数），认不出就不判；列是文本、界也是文本时 +两边写法要一样（'20261012' 对 '2026-10-12' 按字符串比，结果和按日期比对不上），不一样的不判。区间和实际范围完全不重叠才算 +「没有数据」；一列用到几段区间（本周减上周）时每一段都不重叠才算。 + +**和 sqlcheck 一样宁可漏报。** 所有探查都走数据源查询同一道只读守卫（data/guard.check + is_write），遮罩列上的不探查、不报 +值；总耗时有上限（BUDGET_S）；解析不了、认不出、探查失败一律当没问题，出错不影响查询本身（同 tools/datasource._sql_checks）。 +""" +from __future__ import annotations + +import asyncio +import logging +import re +import time +from collections.abc import Awaitable, Callable, Iterable, Mapping +from dataclasses import dataclass, field +from datetime import date, datetime, timedelta +from typing import Any + +from app.data.names import name_key + +logger = logging.getLogger(__name__) + +#: 一次检查（一份查询结果、一条自查的 SQL）所有探查加起来最多多少秒 +BUDGET_S = 20.0 +#: 每条探查查询的时限 +PROBE_TIMEOUT_S = 10.0 +#: 一次最多探查几张表的范围、几条按运行当天现算的界：再多就是 SQL 太复杂，不为它把库查一遍 +MAX_RANGES = 8 +MAX_BOUNDS = 24 +#: 太长的 SQL 不分析（同 sqlcheck.MAX_SQL_CHARS） +MAX_SQL_CHARS = 20_000 +#: 一条 no_data 里最多列几个结果列（计数照实） +MAX_COLUMNS = 12 +#: 现查到的表的范围在进程里记多久（秒）：同一张表一会儿内查好几次（一张图里几个取数节点、助手自查几轮）不再全表 MIN / +#: MAX。滚动表一天才换一次数据，几分钟的旧不影响判断；查不成的、退回剖析的不记 +LIVE_TTL_S = 300.0 +_LIVE: dict[tuple[Any, ...], tuple[float, Any]] = {} +#: 现查没成功（大表全表 MIN / MAX 超时、库一时连不上）也记 LIVE_TTL_S：这段时间里同一张表直接退回剖析、取值观察， +#: 不然取数 agent 连查几次、每次结果里有整列 0，每次都要白等 PROBE_TIMEOUT_S 秒才退回 +_PROBE_FAILED = object() + +#: 给人和模型看的说法 +NO_DATA_MESSAGE = "{columns}查的是 {window}，但 {table} 的 {column} {data}：这些 0 不代表这段时间真的是 0，是这段时间表里没有数据{basis}" +NO_DATA_SQL_MESSAGE = "查的区间是 {window}，但 {table} 的 {column} {data}{basis}" +BASIS_PROFILE = "（现查表的范围没有成功，按 {day} 的剖析）" +BASIS_OBSERVED = "（现查表的范围没有成功，按 {day} 的取值观察）" +#: 写进口径卡、写作目录、双盲复算原因的那半句 +NO_DATA_SHORT = "查询区间里这张表没有数据" + +#: (表达式的类型, 方言) → 按运行当天现算的界：不认列、不认子查询，函数、字面量、INTERVAL 都可以 +_COMPARE = {"GT": ">", "GTE": ">=", "LT": "<", "LTE": "<=", "EQ": "="} +_FLIP = {">": "<", ">=": "<=", "<": ">", "<=": ">=", "=": "="} +_ISO = re.compile(r"^(\d{4})-(\d{1,2})-(\d{1,2})(?:[ T](\d{1,2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?)?" + r"(?:Z|[+-]\d{2}:?\d{2})?$") +_COMPACT = re.compile(r"^(\d{4})(\d{2})(\d{2})$") + + +# ========================================================================== +# 日期的归一 +# ========================================================================== + + +@dataclass(frozen=True) +class _Day: + """归一后的一个日期：day 是哪一天，timed 是带不带一天里的时刻（00:00:00 不算），shape 是写法（iso / compact / + number / date：库交回的是日期类型），kind 是库交回的类型（date / datetime / text / number，字面量按它的写法）。""" + + day: date + timed: bool + shape: str + kind: str + + +def _as_day(value: Any, kind: str | None = None) -> _Day | None: + """一个值归一成日期；认不出返回 None。kind 是库交回的列类型（run_query 的 column_types），字面量不给。""" + if isinstance(value, bool) or value is None: + return None + if isinstance(value, datetime): + return _Day(value.date(), bool(value.time().replace(microsecond=0) != datetime.min.time()), "date", "datetime") + if isinstance(value, date): + return _Day(value, False, "date", "date") + if isinstance(value, (int, float)): + if isinstance(value, float) and not value.is_integer(): + return None + found = _COMPACT.match(str(int(value))) + day = _ymd(found.group(1), found.group(2), found.group(3)) if found else None + return _Day(day, False, "number", kind or "number") if day else None + if not isinstance(value, str): + return None + text = value.strip() + if found := _COMPACT.match(text): + day = _ymd(found.group(1), found.group(2), found.group(3)) + return _Day(day, False, "compact", kind or "text") if day else None + if found := _ISO.match(text): + day = _ymd(found.group(1), found.group(2), found.group(3)) + if day is None: + return None + timed = any(int(found.group(i) or 0) for i in (4, 5, 6)) + shape = "date" if kind in ("date", "datetime") else "iso" + return _Day(day, timed, shape, kind or "text") + return None + + +def _ymd(y: str, m: str, d: str) -> date | None: + try: + return date(int(y), int(m), int(d)) + except ValueError: + return None + + +def _comparable(column: _Day, bound: _Day, *, bare: bool, read: str | None) -> bool: + """库里拿这一列和这个界比，是不是按日期在比（我们的判断才对得上）： + + - 列外面套着函数（DATE(x)、STR_TO_DATE(x, …)）：比的是函数的结果，按日期算； + - 列本身是日期类型：库把文本的界转成日期再比； + - 界是库算出来的日期类型（DATE_SUB(CURRENT_DATE, …)）、列是文本：MySQL 把文本转成日期再比，别的库不一定（SQLite 的 + date('now') 交回的是文本，照下一条比写法），只认 MySQL； + - 两边都是文本、都是数：写法得一样（'20261012' 对 '2026-10-12' 按字符串比，和按日期比对不上）。""" + if not bare or column.kind in ("date", "datetime"): + return True + if bound.kind in ("date", "datetime"): + return read == "mysql" and column.kind == "text" + if column.kind == "text" and bound.kind == "text": + return column.shape == bound.shape + return column.kind == bound.kind == "number" and column.shape == bound.shape == "number" + + +# ========================================================================== +# 从 SQL 里认时间谓词 +# ========================================================================== + + +@dataclass +class _Filter: + """一段 SELECT 里对一张实表的一列的时间条件：table / column 是 sqlglot 的节点（探查时照它写 SQL），conds 是 + [(比较符, 界的节点, 列是不是光着的)]。key 按内容认（表、列、各个条件的写法）：几列各自收窄出来的同一段条件是同一个， + 判一次。""" + + key: tuple[Any, ...] + table: Any + table_name: str + column: Any + column_name: str + conds: list[tuple[str, Any, bool]] = field(default_factory=list) + + +def _read(kind: str | None) -> str | None: + from app.data.sqlcheck import dialect_of + + return dialect_of(kind) + + +def _parse(sql: str, read: str | None) -> Any: + import sqlglot + from sqlglot import exp + + if not isinstance(sql, str) or not sql.strip() or len(sql) > MAX_SQL_CHARS or "{{" in sql: + return None + try: + statements = [s for s in sqlglot.parse(sql, read=read) if s is not None] + except Exception: # noqa: BLE001 - 解析不了就不判 + return None + if len(statements) != 1 or not isinstance(statements[0], exp.Query): + return None + return statements[0] + + +def _output_name(item: Any, read: str | None) -> str: + from sqlglot import exp + + if isinstance(item, exp.Alias): + return item.alias + if isinstance(item, exp.Column): + return item.name + return item.sql(dialect=read) + + +def _same_name(a: str, b: str) -> bool: + return "".join(a.split()).lower() == "".join(b.split()).lower() + + +def narrowed(tree: Any, column: str, read: str | None) -> Any: + """收窄到算出 column 这一列的那一项：最外层 SELECT 只留它，分组、HAVING、排序、取行数去掉（只拿来认时间谓词，不执行， + 不用像双盲复算那样按这一行的分组值补等值条件）。认不准的（不是一段 SELECT、SELECT *、对不上或对上好几项）交回整条。""" + from sqlglot import exp + + if not isinstance(tree, exp.Select): + return tree + items = list(tree.expressions) + if any(isinstance(i, exp.Star) or (isinstance(i, exp.Column) and isinstance(i.this, exp.Star)) for i in items): + return tree + hits = [i for i in items if _same_name(_output_name(i, read), column)] + if len(hits) != 1: + return tree + out = tree.copy() + out.set("expressions", [hits[0].copy()]) + for arg in ("group", "having", "order", "limit", "offset", "qualify"): + out.set(arg, None) + return out + + +def _columns_in(node: Any) -> list[Any]: + from sqlglot import exp + + return [c for c in node.find_all(exp.Column) if not isinstance(c.this, exp.Star)] + + +def _plain_bound(node: Any) -> bool: + """界里没有列、子查询、聚合、窗口：字面量，或者按运行当天现算的表达式。""" + from sqlglot import exp + + return not _columns_in(node) and not any( + isinstance(n, (exp.Select, exp.Subquery, exp.AggFunc, exp.Window, exp.Placeholder, exp.Parameter)) + for n in node.walk()) + + +def _timeish(subject: Any, column: Any, bounds: list[Any]) -> bool: + """像是时间条件：列名像日期、时间，或者列套着日期函数、界里有日期函数、像日期的常量、按运行当天现算的写法。""" + from app.data import sql_semantics as ss + + name = str(column.name or "") + return bool(ss._temporal_name(name) or ss._timeish_name(name) or ss._datey(subject) + or any(ss._datey(b) or ss._date_like_literal(b) or ss._now_based(b) for b in bounds)) + + +def _condition(cond: Any) -> tuple[Any, Any, list[tuple[str, Any]], bool] | None: + """一个条件是不是「一列 比 界」：(列, 列那一侧的表达式, [(比较符, 界)], 列是不是光着的)。BETWEEN、IN（常量列表） + 拆成几项；NOT、子查询、两边都有列的不算。""" + from sqlglot import exp + + from app.data.sqlcheck import _bare_column, _unparen + + n = _unparen(cond) + if n.args.get("negate"): + return None + if isinstance(n, exp.Between): + subject, pairs = n.this, [(">=", n.args.get("low")), ("<=", n.args.get("high"))] + elif isinstance(n, exp.In) and n.expressions and not n.args.get("query"): + subject, pairs = n.this, [("in", e) for e in n.expressions] + elif type(n).__name__ in _COMPARE and isinstance(n, exp.Binary): + op = _COMPARE[type(n).__name__] + left, right = n.this, n.expression + if _columns_in(left) and not _columns_in(right): + subject, pairs = left, [(op, right)] + elif _columns_in(right) and not _columns_in(left): + subject, pairs = right, [(_FLIP[op], left)] + else: + return None + else: + return None + if subject is None or any(b is None or not _plain_bound(b) for _, b in pairs): + return None + cols = _columns_in(subject) + if len(cols) != 1 or any(isinstance(x, (exp.AggFunc, exp.Select, exp.Window)) for x in subject.walk()): + return None + return cols[0], subject, pairs, _bare_column(subject) is not None + + +def _real_tables(select: Any, ctes: set[str]) -> dict[str, Any]: + """这一段 FROM / JOIN 里的实表：{别名或表名的 name_key: 表节点}。CTE、子查询不算。""" + from sqlglot import exp + + sources = [select.args.get("from_") or select.args.get("from")] + list(select.args.get("joins") or []) + out: dict[str, Any] = {} + for source in sources: + table = getattr(source, "this", None) + if isinstance(table, exp.Table) and table.name and name_key(table.name) not in ctes: + out[name_key(table.alias_or_name)] = table + return out + + +def _schema_columns(schema: Mapping[str, Any] | None, table: Any) -> set[str] | None: + """表结构里这张表的列（name_key）；表结构里没有这张表为 None。""" + tables = (schema or {}).get("tables") if isinstance(schema, Mapping) else None + if not isinstance(tables, Mapping): + return None + for name, meta in tables.items(): + if name_key(str(name).rsplit(".", 1)[-1]) == name_key(table.name) and isinstance(meta, Mapping): + return {name_key(str(c.get("name"))) for c in meta.get("columns") or [] if isinstance(c, Mapping)} + return None + + +def filters(tree: Any, schema: Mapping[str, Any] | None = None) -> list[_Filter]: + """语法树里各段 SELECT 的 WHERE 中，对实表的一列的时间条件，按（段、表、列）归在一起。""" + from sqlglot import exp + + from app.data.sqlcheck import _conjuncts + + ctes = {name_key(c.alias_or_name) for c in tree.find_all(exp.CTE) if c.alias_or_name} + found: dict[tuple[int, str, str], _Filter] = {} + for index, select in enumerate(tree.find_all(exp.Select)): + where = select.args.get("where") + if where is None: + continue + tables = _real_tables(select, ctes) + if not tables: + continue + for cond in _conjuncts(where.this): + parsed = _condition(cond) + if parsed is None: + continue + column, subject, pairs, bare = parsed + if not _timeish(subject, column, [b for _, b in pairs]): + continue + if column.table: + table = tables.get(name_key(column.table)) + elif len(tables) == 1: + table = next(iter(tables.values())) + else: + owners = [t for t in tables.values() + if name_key(column.name) in (_schema_columns(schema, t) or set())] + table = owners[0] if len(owners) == 1 else None + if table is None: + continue + shown = ".".join(p for p in (table.catalog, table.db, table.name) if p) + group = (index, name_key(shown), name_key(column.name)) + entry = found.setdefault(group, _Filter(group, table, shown, column, str(column.name))) + entry.conds.extend((op, bound, bare) for op, bound in pairs) + for entry in found.values(): + entry.key = (name_key(entry.table_name), name_key(entry.column_name), + tuple(sorted((op, bound.sql(), bare) for op, bound, bare in entry.conds))) + return list(found.values()) + + +# ========================================================================== +# 探查 +# ========================================================================== + + +async def _probe(source: Any, sql: str) -> tuple[list[Any], list[str | None]] | None: + """在数据源上跑一条探查查询：第一行的值、各列的类型（按位置）。走数据源查询同一道只读守卫；写语句、出错、超时都交回 + None。""" + from app.data.engine import run_query + from app.data.guard import QueryLimits, SqlRejected, check, is_write + + kind = (getattr(source, "kind", None) or "").lower() + try: + statement = check(sql, readonly=True, source_name=getattr(source, "name", ""), dialect=kind) + if is_write(statement, dialect=kind): + return None + result = await run_query(source, statement, limits=QueryLimits(timeout_seconds=PROBE_TIMEOUT_S, max_rows=2)) + except (SqlRejected, Exception): # noqa: BLE001 - 探查失败就当不知道 + logger.info("覆盖检查的探查查询没有成功（数据源 %s）", getattr(source, "name", ""), exc_info=True) + return None + if not result.rows: + return None + types = result.column_types or {} + return list(result.rows[0]), [types.get(c) for c in result.columns] + + +def _select_sql(items: list[str], read: str | None) -> str: + return f"SELECT {', '.join(items)}" + (" FROM DUAL" if read == "oracle" else "") + + +@dataclass +class Cache: + """同一个数据源上探查过的：表里那一列的范围、按运行当天现算的界。助手自查一轮里查好几次，不重复查库。""" + + ranges: dict[tuple[str, str, str], Any] = field(default_factory=dict) + bounds: dict[tuple[str, str], Any] = field(default_factory=dict) + + +@dataclass +class _Range: + """这一列在这张表里的范围。low / high 为 None 的是不知道那一头；empty 是表里这一列一个值都没有（现查出来的）。""" + + low: _Day | None + high: _Day | None + basis: str + day: str | None = None + empty: bool = False + + +#: 剖析事实（catalog_profile_jobs.last_facts 的一项）：表名 → 那一份，取不到为 None +Facts = Callable[[str], Awaitable[Mapping[str, Any] | None]] + + +async def _range(source: Any, flt: _Filter, read: str | None, cache: Cache, facts: Facts | None) -> _Range | None: + """这一列在这张表里的范围：现查 MIN / MAX（同一个数据源、同一张表、同一列查一次），查不成退回剖析、取值观察（facts）。""" + from sqlglot import exp + + key = (str(getattr(source, "name", "")), name_key(flt.table_name), name_key(flt.column_name)) + if key in cache.ranges: + return cache.ranges[key] + live = (getattr(source, "id", None), getattr(source, "snapshot_id", None), *key) + held = _LIVE.get(live) + failed_before = False + if held is not None and held[0] > time.monotonic(): + if held[1] is not _PROBE_FAILED: + cache.ranges[key] = held[1] + return held[1] + failed_before = True + table = flt.table.copy() + table.set("alias", None) + ident = flt.column.this.copy() if isinstance(flt.column.this, exp.Identifier) \ + else exp.to_identifier(flt.column_name) + column = exp.Column(this=ident) + sql = (f"SELECT {exp.Min(this=column.copy()).sql(dialect=read)} AS lo, " + f"{exp.Max(this=column.copy()).sql(dialect=read)} AS hi FROM {table.sql(dialect=read)}") + found = None if failed_before else await _probe(source, sql) + out: _Range | None = None + if found is not None: + values, kinds = found + low_raw, high_raw = (values + [None, None])[:2] + kinds = (kinds + [None, None])[:2] + if low_raw is None and high_raw is None: + out = _Range(None, None, "live", empty=True) + else: + low, high = _as_day(low_raw, kinds[0]), _as_day(high_raw, kinds[1]) + out = _Range(low, high, "live") if low is not None and high is not None else None + elif facts is not None: + out = await _fallback(flt, facts) + cache.ranges[key] = out + if found is None and not failed_before: + _remember(live, _PROBE_FAILED) + elif out is not None and out.basis == "live": + _remember(live, out) + return out + + +def _remember(live: tuple[Any, ...], value: Any) -> None: + if len(_LIVE) > 512: + _LIVE.clear() + _LIVE[live] = (time.monotonic() + LIVE_TTL_S, value) + + +async def _fallback(flt: _Filter, facts: Facts) -> _Range | None: + """现查不成：退回剖析记下的范围（date_ranges），再不然取值观察的最新日期（freshness，只知道最晚那一头）。""" + try: + found = await facts(flt.table_name) + except Exception: # noqa: BLE001 + return None + if not isinstance(found, Mapping): + return None + wanted = name_key(flt.column_name) + for item in found.get("date_ranges") or []: + if isinstance(item, Mapping) and name_key(str(item.get("column") or "")) == wanted: + lo, hi = _as_day(item.get("min")), _as_day(item.get("max")) + if lo is not None and hi is not None: + return _Range(lo, hi, "profile", day=str(found.get("at") or "")[:10] or None) + observed = found.get("observed") if isinstance(found.get("observed"), Mapping) else {} + for item in observed.get("freshness") or []: + if isinstance(item, Mapping) and name_key(str(item.get("column") or "")) == wanted: + hi = _as_day(item.get("max")) + if hi is not None: + return _Range(None, hi, "observed", day=str(item.get("at") or "")[:10] or None) + return None + + +def _literal_day(node: Any) -> _Day | None: + from sqlglot import exp + + from app.data.sqlcheck import _unparen + + node = _unparen(node) + negative = False + if isinstance(node, exp.Neg): + node, negative = _unparen(node.this), True + if not isinstance(node, exp.Literal) or negative: + return None + if node.is_string: + return _as_day(str(node.this)) + try: + value = int(str(node.this)) + except ValueError: + return None + return _as_day(value) + + +def _is_literal(node: Any) -> bool: + from sqlglot import exp + + from app.data.sqlcheck import _unparen + + node = _unparen(node) + if isinstance(node, exp.Neg): + node = _unparen(node.this) + return isinstance(node, exp.Literal) + + +async def _bounds(source: Any, nodes: list[Any], read: str | None, cache: Cache) -> dict[int, _Day | None]: + """各个界归一成日期：字面量直接解析，别的（按运行当天现算的表达式）一条 SELECT 在库里算出来。{id(节点): 日期}。""" + out: dict[int, _Day | None] = {} + pending: dict[str, list[int]] = {} + name = str(getattr(source, "name", "")) + for node in nodes: + if _is_literal(node): + out[id(node)] = _literal_day(node) + continue + try: + text = node.sql(dialect=read) + except Exception: # noqa: BLE001 + out[id(node)] = None + continue + if (name, text) in cache.bounds: + out[id(node)] = cache.bounds[(name, text)] + else: + pending.setdefault(text, []).append(id(node)) + texts = list(pending)[:MAX_BOUNDS] + found = await _probe(source, _select_sql([f"{t} AS b{i}" for i, t in enumerate(texts)], read)) if texts else None + for i, text in enumerate(pending): + day = None + if found is not None and i < len(texts): + values, kinds = found + day = _as_day(values[i], kinds[i]) if i < len(values) and i < len(kinds) else None + if i < len(texts): + cache.bounds[(name, text)] = day + for ident in pending[text]: + out[ident] = day + return out + + +@dataclass(frozen=True) +class Window: + """一段时间条件的范围（按天，两头都含）。start / end 为 None 的是那一头没有限制。""" + + start: date | None + end: date | None + + +def _window(conds: list[tuple[str, Any, bool]], days: Mapping[int, _Day | None], data: _Range, + read: str | None) -> Window | None: + """几个条件合成一段：下界取最晚的、上界取最早的；IN 列表取最早到最晚（中间没列到的日子也算进去，宁可判成有重叠）。 + 有一个界认不出、或者和列比不起来，整段不判（None）。「>」「<」的界不带时刻时按天挪一天（> 2026-10-11 从 10-12 算起）； + 带时刻的不挪（那一天里还可能有数据，宁可判成有重叠）。""" + lows: list[date] = [] + highs: list[date] = [] + points: list[date] = [] + sample = data.high or data.low + for op, node, bare in conds: + day = days.get(id(node)) + if day is None or (sample is not None and not _comparable(sample, day, bare=bare, read=read)): + return None + if op == "in": + points.append(day.day) + continue + if op in (">=", "="): + lows.append(day.day) + elif op == ">": + lows.append(day.day if day.timed else day.day + timedelta(days=1)) + if op in ("<=", "="): + highs.append(day.day) + elif op == "<": + highs.append(day.day if day.timed else day.day - timedelta(days=1)) + if points: + lows.append(min(points)) + highs.append(max(points)) + start = max(lows) if lows else None + end = min(highs) if highs else None + if start is None and end is None: + return None + if start is not None and end is not None and start > end: + return None + return Window(start, end) + + +def _disjoint(window: Window, data: _Range) -> bool: + """区间和表里的范围完全不重叠（表里这一列一个值都没有也算）。""" + if data.empty: + return True + if window.end is not None and data.low is not None and window.end < data.low.day: + return True + return window.start is not None and data.high is not None and window.start > data.high.day + + +# ========================================================================== +# 说法 +# ========================================================================== + + +def _window_text(window: Window) -> str: + if window.start is not None and window.end is not None: + return window.start.isoformat() if window.start == window.end \ + else f"{window.start.isoformat()} 到 {window.end.isoformat()}" + if window.start is not None: + return f"{window.start.isoformat()} 起" + return f"{window.end.isoformat()} 及以前" # type: ignore[union-attr] + + +def _data_text(data: _Range) -> str: + if data.empty: + return "一个值都没有" + if data.low is not None and data.high is not None: + if data.low.day == data.high.day: + return f"只有 {data.low.day.isoformat()} 这一天的数据" + return f"只有 {data.low.day.isoformat()} 到 {data.high.day.isoformat()} 的数据" + return f"最新只到 {data.high.day.isoformat()}" # type: ignore[union-attr] + + +def _basis_text(data: _Range) -> str: + if data.basis == "profile": + return BASIS_PROFILE.format(day=data.day or "上次") + if data.basis == "observed": + return BASIS_OBSERVED.format(day=data.day or "上次") + return "" + + +def _columns_text(columns: list[str]) -> str: + head = f"「{columns[0]}」" + return f"{head}等 {len(columns)} 列" if len(columns) > 1 else head + + +def _range_dict(data: _Range) -> list[str | None] | None: + if data.empty: + return None + return [data.low.day.isoformat() if data.low else None, data.high.day.isoformat() if data.high else None] + + +def _window_dict(window: Window) -> list[str | None]: + return [window.start.isoformat() if window.start else None, window.end.isoformat() if window.end else None] + + +# ========================================================================== +# 入口 +# ========================================================================== + + +def zero_columns(payload: Mapping[str, Any]) -> list[str]: + """查询结果里「整列都是 0 或空」的数值列（按 column_types 是数的、或者没记类型却全是数和空值的）。一行都没有的结果 + 返回空列表：那归「没有查到」管，不归这里。""" + from app.engine.expressions import numeric_text + + columns = [str(c) for c in payload.get("columns") or []] + rows = payload.get("rows") or [] + if not columns or not isinstance(rows, list) or not rows: + return [] + types = payload.get("column_types") if isinstance(payload.get("column_types"), Mapping) else {} + out: list[str] = [] + for index, column in enumerate(columns): + if types.get(column) not in (None, "number"): + continue + values = [row[index] if isinstance(row, (list, tuple)) and index < len(row) + else (row.get(column) if isinstance(row, Mapping) else None) for row in rows] + + def zero(value: Any) -> bool: + if value is None: + return True + if isinstance(value, bool): + return False + if isinstance(value, (int, float)): + return value == 0 + number = numeric_text(value) if isinstance(value, str) else None + return number is not None and number == 0 + + if all(zero(v) for v in values): + out.append(column) + return out + + +async def _judge(source: Any, flts: list[_Filter], read: str | None, cache: Cache, facts: Facts | None, + masked: set[str], deadline: float) -> dict[tuple[int, str, str], tuple[str, Window | None, _Range | None]]: + """每段时间条件判一下：none（区间里表里没有数据）/ some（有重叠）/ unknown（判断不了），带上区间和范围。""" + out: dict[tuple[int, str, str], tuple[str, Window | None, _Range | None]] = {} + ranges: dict[tuple[str, str], _Range | None] = {} + usable = [f for f in flts if name_key(f.column_name) not in masked] + for flt in flts: + if flt not in usable: + out[flt.key] = ("unknown", None, None) + for flt in usable: + if time.monotonic() > deadline: + out[flt.key] = ("unknown", None, None) + continue + rkey = (name_key(flt.table_name), name_key(flt.column_name)) + if rkey not in ranges: + ranges[rkey] = await _range(source, flt, read, cache, facts) if len(ranges) < MAX_RANGES else None + data = ranges[rkey] + if data is None: + out[flt.key] = ("unknown", None, None) + continue + days = await _bounds(source, [node for _, node, _ in flt.conds], read, cache) + window = _window(flt.conds, days, data, read) + if window is None: + out[flt.key] = ("unknown", None, data) + else: + out[flt.key] = ("none" if _disjoint(window, data) else "some", window, data) + return out + + +async def result_gaps(source: Any, sql: str, payload: Mapping[str, Any], *, facts: Facts | None = None, + masked: Iterable[str] = (), cache: Cache | None = None, + budget: float = BUDGET_S) -> list[dict[str, Any]]: + """一份查询结果里，整列是 0 或空、而查询区间里这张表没有数据的那几列：[{columns, table, column, window, data_range, + basis, basis_at?, message}]，同一张表、同一列、同一段区间的几列并成一条。没有问题（或者判断不了）返回空列表。从不抛异常。""" + try: + return await asyncio.wait_for(_result_gaps(source, sql, payload, facts, set(masked), cache or Cache(), budget), + timeout=budget + PROBE_TIMEOUT_S) + except asyncio.TimeoutError: + logger.warning("覆盖检查超过时限，本次跳过（数据源 %s）", getattr(source, "name", "")) + except Exception: # noqa: BLE001 - 检查不能让一次成功的查询变成失败 + logger.exception("覆盖检查出错，本次跳过（数据源 %s）", getattr(source, "name", "")) + return [] + + +async def _result_gaps(source: Any, sql: str, payload: Mapping[str, Any], facts: Facts | None, masked: set[str], + cache: Cache, budget: float) -> list[dict[str, Any]]: + zeros = zero_columns(payload) + if not zeros: + return [] + read = _read(getattr(source, "kind", None)) + tree = _parse(sql, read) + if tree is None: + return [] + masked = {m.strip().lower() for m in masked if isinstance(m, str) and m.strip()} + schema = getattr(source, "schema_cache", None) + per_column = {c: filters(narrowed(tree, c, read), schema) for c in zeros if c.lower() not in masked} + every = {f.key: f for flts in per_column.values() for f in flts} + if not every: + return [] + judged = await _judge(source, list(every.values()), read, cache, facts, masked, time.monotonic() + budget) + groups: dict[tuple[Any, ...], dict[str, Any]] = {} + for column, flts in per_column.items(): + verdicts = [judged.get(f.key) for f in flts] + if not flts or any(v is None or v[0] != "none" for v in verdicts): + continue + for flt, (_, window, data) in zip(flts, verdicts): + gkey = (name_key(flt.table_name), name_key(flt.column_name), window, data.basis if data else None) + group = groups.setdefault(gkey, {"flt": flt, "window": window, "data": data, "columns": []}) + if column not in group["columns"]: + group["columns"].append(column) + out = [] + for group in groups.values(): + flt, window, data, columns = group["flt"], group["window"], group["data"], group["columns"] + out.append({ + "columns": columns[:MAX_COLUMNS], "table": flt.table_name, "column": flt.column_name, + "window": _window_dict(window), "data_range": _range_dict(data), "basis": data.basis, + **({"basis_at": data.day} if data.day else {}), + "message": NO_DATA_MESSAGE.format(columns=_columns_text(columns), window=_window_text(window), + table=flt.table_name, column=flt.column_name, + data=_data_text(data), basis=_basis_text(data)), + }) + return out + + +async def sql_gaps(source: Any, sql: str, *, facts: Facts | None = None, masked: Iterable[str] = (), + cache: Cache | None = None, budget: float = BUDGET_S) -> list[dict[str, Any]]: + """还没执行的一条 SQL（助手自查）：SQL 里每一段时间条件，区间和表里的范围完全不重叠的那几段：[{table, column, + window, data_range, basis, basis_at?, message}]，同一张表、同一列、同一段区间只算一条。从不抛异常。""" + try: + return await asyncio.wait_for(_sql_gaps(source, sql, facts, set(masked), cache or Cache(), budget), + timeout=budget + PROBE_TIMEOUT_S) + except asyncio.TimeoutError: + logger.warning("自查的覆盖检查超过时限，本次跳过（数据源 %s）", getattr(source, "name", "")) + except Exception: # noqa: BLE001 + logger.exception("自查的覆盖检查出错，本次跳过（数据源 %s）", getattr(source, "name", "")) + return [] + + +async def _sql_gaps(source: Any, sql: str, facts: Facts | None, masked: set[str], cache: Cache, + budget: float) -> list[dict[str, Any]]: + read = _read(getattr(source, "kind", None)) + tree = _parse(sql, read) + if tree is None: + return [] + masked = {m.strip().lower() for m in masked if isinstance(m, str) and m.strip()} + flts = filters(tree, getattr(source, "schema_cache", None)) + if not flts: + return [] + judged = await _judge(source, flts, read, cache, facts, masked, time.monotonic() + budget) + out: list[dict[str, Any]] = [] + seen: set[tuple[Any, ...]] = set() + for flt in flts: + verdict, window, data = judged.get(flt.key, ("unknown", None, None)) + key = (name_key(flt.table_name), name_key(flt.column_name), window) + if verdict != "none" or window is None or data is None or key in seen: + continue + seen.add(key) + out.append({"table": flt.table_name, "column": flt.column_name, "window": _window_dict(window), + "data_range": _range_dict(data), "basis": data.basis, **({"basis_at": data.day} if data.day else {}), + "message": NO_DATA_SQL_MESSAGE.format(window=_window_text(window), table=flt.table_name, + column=flt.column_name, data=_data_text(data), + basis=_basis_text(data))}) + return out + + +# ========================================================================== +# 读快照里记下的 +# ========================================================================== + + +def no_data_of(snapshot: Any, column: Any) -> dict[str, Any] | None: + """查询快照里记着 column 这一列「查询区间里这张表没有数据」的那一条（no_data）；没有为 None。按列名认（不分大小写、 + 不计空白）。""" + if not isinstance(snapshot, Mapping) or not isinstance(column, str): + return None + for item in snapshot.get("no_data") or []: + if isinstance(item, Mapping) and any(isinstance(c, str) and _same_name(c, column) + for c in item.get("columns") or []): + return dict(item) + return None + + +__all__ = ["BUDGET_S", "Cache", "NO_DATA_SHORT", "PROBE_TIMEOUT_S", "Window", "filters", "narrowed", "no_data_of", + "result_gaps", "sql_gaps", "zero_columns"] diff --git a/backend/app/data/schema_view.py b/backend/app/data/schema_view.py index 1b3ef7cb..6baed35d 100644 --- a/backend/app/data/schema_view.py +++ b/backend/app/data/schema_view.py @@ -69,6 +69,8 @@ PROFILE_ROWS_PROMPT = {"stats": "约 {n} 行（数据库统计估算）", "count "at_least": "至少 {n} 行（数到上限为止）"} PROFILE_RANGE_PROMPT = "{column} 从 {low} 到 {high}" PROFILE_HINT_PROMPT = "只供写查询参考（定时间范围、估表大小），不是查询结果，不能作为回答里的数" +#: 日期范围最早和最晚是同一天：这张表很可能只存最新一天（每天整表重灌），拿它算上周、上月只会查出 0 +PROFILE_ONE_DAY_PROMPT = "剖析时这张表的 {column} 只有一天（{day}）的数据，可能每天整表重灌：算历史周期请换有历史的表" PROFILE_STALE_PROMPT = "数据剖析：剖析早于当前版本，不给" #: 取值观察的结论（next-round 03 B2）：最新业务日期；行数比剖析的新时用观察的 PROFILE_LATEST_PROMPT = "最新业务日期 {column} 到 {value}（取值观察，{day}）" @@ -241,6 +243,7 @@ def profile_lines(facts: Mapping[str, Any] | None, masked: set[str], snapshot_id if upload and (not snapshot_id or facts.get("snapshot_id") != snapshot_id): return [indent + PROFILE_STALE_PROMPT] parts: list[str] = [] + one_day: list[str] = [] estimate = facts.get("row_estimate") if isinstance(facts.get("row_estimate"), Mapping) else None seen_rows = observed.get("rows") if isinstance(observed.get("rows"), Mapping) else None if seen_rows and str(seen_rows.get("at") or "") > str(facts.get("at") or ""): @@ -259,6 +262,8 @@ def profile_lines(facts: Mapping[str, Any] | None, masked: set[str], snapshot_id if not column or column.lower() in masked or found.get("min") is None or found.get("max") is None: continue parts.append(PROFILE_RANGE_PROMPT.format(column=column, low=_when(found["min"]), high=_when(found["max"]))) + if (day := _same_day(found["min"], found["max"])) is not None: + one_day.append(PROFILE_ONE_DAY_PROMPT.format(column=column, day=day)) for found in observed.get("freshness") or []: column = str((found or {}).get("column") or "") if column and column.lower() not in masked and found.get("max"): @@ -271,7 +276,15 @@ def profile_lines(facts: Mapping[str, Any] | None, masked: set[str], snapshot_id # 只有取值观察、没剖析过：照实说是观察 latest = max((str(f.get("at") or "") for f in [seen_rows or {}, *(observed.get("freshness") or [])]), default="") head = OBSERVE_HEAD_PROMPT.format(day=_day(latest), facts="；".join(parts)) - return [indent + head, indent + " " + PROFILE_HINT_PROMPT] + return [indent + head, *(indent + " " + line for line in one_day), indent + " " + PROFILE_HINT_PROMPT] + + +def _same_day(low: Any, high: Any) -> str | None: + """日期范围的两头是同一天（DATE、DATETIME、'YYYY-MM-DD'、'YYYYMMDD' 都认，data/coverage 同一个归一）：那一天，不是为 None。""" + from app.data.coverage import _as_day + + lo, hi = _as_day(low), _as_day(high) + return lo.day.isoformat() if lo is not None and hi is not None and lo.day == hi.day else None # ========================================================================== diff --git a/backend/app/engine/compare.py b/backend/app/engine/compare.py index bbc278ae..5481382e 100644 --- a/backend/app/engine/compare.py +++ b/backend/app/engine/compare.py @@ -201,6 +201,16 @@ class Side: inputs: dict[str, Any] = field(default_factory=dict) +def missing_reason(side: str, metric: dict[str, Any]) -> str | None: + """一边的指标没有值时的那句原因：输入都齐、计算式自己算不出来的（口径卡记了 undefined，分母为 0）说「算不出来」， + 别的说「缺少输入」。有值的为 None。""" + if metric.get("status") != "missing_input": + return None + if metric.get("undefined"): + return f"{side}这个指标算不出来（这组输入下计算式没有值，比如分母为 0）" + return f"{side}这个指标缺少输入，没有值" + + def compare(base: Side, target: Side) -> dict[str, Any]: """对比两次运行（base 是基准，target 是这次），只读，不碰两次运行的任何东西。返回： @@ -1499,8 +1509,8 @@ class _Compare: out.update(classification=DATA_ONLY, reasons=[ DATA_FROM_UPSTREAM if self.template["same_graph"] else DATA_FROM_UPSTREAM_SCOPED]) for side, metric in (("基准运行", bm), ("这次运行", tm)): - if metric.get("status") == "missing_input": - out["reasons"].append(f"{side}这个指标缺少输入，没有值") + if said := missing_reason(side, metric): + out["reasons"].append(said) if out["classification"] in _ATTRIBUTED and bp is not None and tp is not None: out["attribution"] = self._attribution(bp, tp, card_id) if any(inp.masked for plan in (bp, tp) if plan is not None for inp in plan.inputs.values()): diff --git a/backend/app/engine/evidence.py b/backend/app/engine/evidence.py index 66dcea28..b06b96e7 100644 --- a/backend/app/engine/evidence.py +++ b/backend/app/engine/evidence.py @@ -926,6 +926,12 @@ def build_catalog( # 来源查询没通过 SQL 检查（data/sqlcheck.py 的 error 级问题）：同样只在有问题时有这两个键 **({"sql_check_failed": True, "sql_check_reason": metric.get("sql_check_reason") or ""} if metric.get("sql_check_failed") else {}), + # 输入都齐、计算式自己算不出来（分母为 0）：写作目录、引用解析不了的原因说「算不出来」，不说「缺少输入」 + **({"undefined": True} if metric.get("undefined") and metric.get("value") is None else {}), + # 输入取自「查询区间里这张表没有数据」的列（data/coverage.py）：写作目录里提醒写作者，证据面板标出来。 + # 只在有问题时有这两个键 + **({"no_data": True, "no_data_reason": metric.get("no_data_reason") or ""} + if metric.get("no_data") else {}), } queries = retrievals = 0 @@ -1653,7 +1659,8 @@ def resolve_marker(marker: dict[str, Any], catalog: dict[str, Any], *, cells_all # 换算了只会解析不了、原因说成「不是数值」。引用它本身由报告自查另记一条违规（masked_metrics） rendered = str(entry.get("rendered") or "") elif entry.get("value") is None: - return _unresolved(marker, alias, ev_kind, f"指标「{entry.get('name') or ref}」本次没有值（缺少输入），" + why = UNDEFINED_REASON if entry.get("undefined") else "缺少输入" + return _unresolved(marker, alias, ev_kind, f"指标「{entry.get('name') or ref}」本次没有值（{why}），" "无法写入报告", rendered=MISSING) else: try: @@ -3450,6 +3457,17 @@ SEE_AND_CODE_RULES = ( "（「状态为 9」「类型 1」）：码值是没有出处的数字，系统会当成裸数字打回重写。" ) +#: 查询区间里这张表没有数据（查询快照的 no_data、口径卡的 no_data，data/coverage.py）：写作目录里的两种提醒、写作规则。 +#: 都带着 NO_DATA_MARK 这几个字：报告节点据此决定要不要加 NO_DATA_RULE +NO_DATA_MARK = "这段时间表里没有数据" +NO_DATA_METRIC_NOTE = "（这段时间表里没有数据：所依据的查询区间在表里查不到数据，这个数不要当成 0 解读，照实写没有数据）" +NO_DATA_QUERY_NOTE = " （注意：{message}。引用这几列时照实写这段时间表里没有数据，不要当成 0 解读）" +NO_DATA_RULE = ("标着「这段时间表里没有数据」的指标和查询列：不要写成 0、持平或 0%，照实写这段时间表里没有数据、" + "数据只到哪天（写具体日期）。") + +#: 指标「算不出来」的那句（口径卡记了 undefined：输入都齐、计算式自己交回空值）：引用解析不了的原因、写作目录里用 +UNDEFINED_REASON = "算不出来：这组输入下计算式没有值，比如分母为 0" + #: [[m:]]、[[v:]] 渲染出来自带单位（部署上撞过「覆盖 8条 条产线」：口径卡的单位是「条」，写作者照着助手的指令又写了一个）。 #: 渲染时紧跟着的同一个单位会去掉（_tidy），这里先说给写作者听 UNIT_RULE = ("数字标记渲染出来自带单位（[[m:…]] 是口径卡里的单位，比如「8条」「12,345人次」），后面不要再写一遍单位：" @@ -3725,6 +3743,10 @@ def _query_prompt(entry: dict[str, Any], snaps: _Snapshots, cells_allowed: bool, "不要把取回的行数当成总数，也不要据此写合计、全部之类的结论）") if note := _sql_check_note(entry.get("artifact"), snaps): lines.append(note) + for gap in snapshot.get("no_data") or [] if isinstance(snapshot.get("no_data"), list) else []: + # 查询区间里这张表没有数据（data/coverage.py）：这几列的 0 不是真的是 0 + if isinstance(gap, dict) and gap.get("message"): + lines.append(NO_DATA_QUERY_NOTE.format(message=gap["message"])) if not cells_allowed: return lines columns = [str(c) for c in snapshot.get("columns") or []][:_PROMPT_COLS] @@ -3838,8 +3860,13 @@ def _metric_prompt(catalog: dict[str, Any]) -> list[str]: if e.get("masked"): # 值取自遮罩列的一格（withhold_metrics）：写作者看到的就是「已遮罩」，引用了也只会交回重写 lines.append(f"- [[{e['alias']}]] {e.get('name')} = {e.get('rendered')}（取自设置了遮罩的列，不要引用）") + elif e.get("value") is None and e.get("undefined"): + # 输入都齐、计算式自己算不出来（分母为 0）：照实写算不出来，不写成 0、持平 + lines.append(f"- [[{e['alias']}]] {e.get('name')}：这次算不出来（{UNDEFINED_REASON}），不要引用；" + "照实写算不出来，不要写成 0 或持平" + (NO_DATA_METRIC_NOTE if e.get("no_data") else "")) elif e.get("value") is None: - lines.append(f"- [[{e['alias']}]] {e.get('name')}：这次没有值，不要引用") + lines.append(f"- [[{e['alias']}]] {e.get('name')}：这次没有值，不要引用" + + (NO_DATA_METRIC_NOTE if e.get("no_data") else "")) elif e.get("rendered") == MISSING: # 有值但按口径卡的格式显示不出来（太小、太大、不是有限的数）：引用了也是解析不了 lines.append(f"- [[{e['alias']}]] {e.get('name')}：按口径卡的格式显示不出来，不要引用") @@ -3850,6 +3877,8 @@ def _metric_prompt(catalog: dict[str, Any]) -> list[str]: # 取数的 SQL 没通过检查（重复计算、存量跨期加总）：数是算错的可能很大，写作者引用时得交代 if e.get("sql_check_failed"): caveat += "（所依据的查询未通过 SQL 检查，结果存疑：引用时须如实说明，不要当作确定的结论）" + if e.get("no_data"): + caveat += NO_DATA_METRIC_NOTE lines.append(f"- [[{e['alias']}]] {e.get('name')} = {e.get('rendered')}{caveat}") inputs = [e for e in catalog.values() if e.get("kind") == "input"] if inputs: diff --git a/backend/app/engine/issuance.py b/backend/app/engine/issuance.py index 4f389a68..bd87cfc8 100644 --- a/backend/app/engine/issuance.py +++ b/backend/app/engine/issuance.py @@ -475,6 +475,15 @@ def incomplete_gaps(metrics: list[dict[str, Any]]) -> list[str]: for m in metrics if isinstance(m, dict) and m.get("incomplete")] +def no_data_gaps(metrics: list[dict[str, Any]]) -> list[str]: + """契约收来的指标里，输入取自「查询区间里这张表没有数据」的列的（口径卡标了 no_data，data/coverage.py），每个一句缺口。 + + 数回指得上口径卡、复算也一致，可那个 0 是因为这段时间表里根本没有数据，不是真的是 0：盖完整出具的章等于替「上周到访 + 0 人」作保。原因用口径卡记下的那句（查询记下的区间和表里的范围）。""" + return [f"指标「{m.get('name') or m.get('id')}」{m.get('no_data_reason') or '所依据的查询区间里这张表没有数据'}" + for m in metrics if isinstance(m, dict) and m.get("no_data")] + + def sql_check_gaps(metrics: list[dict[str, Any]], cited: dict[str, list[str]] | None = None, *, label: Callable[[str, str | None], str | None] | None = None) -> list[str]: """sql_check_findings 的那几句缺口（见下）。""" diff --git a/backend/app/engine/nodes/dual_check.py b/backend/app/engine/nodes/dual_check.py index 642cd847..9ce7e610 100644 --- a/backend/app/engine/nodes/dual_check.py +++ b/backend/app/engine/nodes/dual_check.py @@ -69,6 +69,7 @@ from app.core.events import EventType from app.db.base import SessionLocal from app.db.models import DataSource from app.data import catalog_view +from app.data.coverage import NO_DATA_SHORT, no_data_of from app.engine import deadline, recompute from app.engine.approval import read_decision from app.engine.context import NodeContext, NodeError, NodeRejected @@ -635,6 +636,10 @@ async def _query_input(item: dict[str, Any], sources: _Sources, aliases: dict[st hidden = await sources.hidden(artifact) if column.lower() in hidden: return {}, f"输入 {item['path']} 取自设置了遮罩的列，不能用于复算" + if (gap := no_data_of(snap, column)) is not None: + # 原值所在的那一格落在「查询区间里这张表没有数据」的列上（data/coverage.py）：那个 0 不代表真的是 0，盲算查到 0 + # 也说明不了一致，不白派一次盲算，按无法复算记 + return {}, f"{NO_DATA_SHORT}（输入 {item['path']}：{gap.get('message') or ''}）" value = cell_value(item.get("value")) if not _number(value): return {}, f"输入 {item['path']} 不是数值" @@ -1003,6 +1008,30 @@ def _settled(found: Mapping[str, Any]) -> bool: return found.get("limit") == "max_queries" and found.get("value") is not None +#: 无法复算的原因里带上盲算交回的理由时最多截多少字（理由是渲染后的正文，已经过遮罩） +RATIONALE_CHARS = 160 + + +def _rationale_tail(found: Mapping[str, Any]) -> str: + """盲算触顶、按无法复算记时（_limit_text 那一路），它交回的理由截一段接在原因后面：审批卡、缺口说明上看得到它查到了 + 什么。没有理由的交回空串（原因照旧）。""" + text = " ".join(str(found.get("rationale") or "").split()) + if not text: + return "" + return f"；盲算交回的理由：{text[:RATIONALE_CHARS]}{'…' if len(text) > RATIONALE_CHARS else ''}" + + +def _no_data_cell(artifact: Any, column: Any, load: Callable[[str], Any]) -> dict[str, Any] | None: + """这一格所在的列「查询区间里这张表没有数据」（查询快照的 no_data，data/coverage.py）：那一条，不是为 None。""" + if not isinstance(artifact, str) or not artifact or not isinstance(column, str): + return None + try: + snapshot = load(artifact) + except (ValueError, OSError): + return None + return no_data_of(snapshot, column) + + def _limit_text(limit: str | None, options: _Options) -> str: """盲算为什么停下，接在「盲算」后面读：「盲算已达每项最多查询次数（6 次）」。""" if limit == "max_queries" and options.max_queries is not None: @@ -1027,12 +1056,17 @@ def _compare(row: dict[str, Any], metric: dict[str, Any], results: dict[str, Any row.update(status=UNCHECKED, reason="盲算没有执行") return if found.get("limit") and not _settled(found): - row.update(status=UNCHECKED, reason="盲算" + _limit_text(found["limit"], options)) + row.update(status=UNCHECKED, reason="盲算" + _limit_text(found["limit"], options) + _rationale_tail(found)) return if found.get("value") is None: row.update(status=UNCHECKED, reason=found.get("reason") or "盲算没有交回可用的数值", **{k: found[k] for k in ("cause", "run_seconds") if k in found}) # 运行时长上限将至（_blind） return + if gap := _no_data_cell(found.get("artifact"), found.get("column"), load): + # 盲算交回的那一格落在「查询区间里这张表没有数据」的列上（data/coverage.py）：那个 0 不代表真的是 0，拿它和原值 + # 比一致不一致都说明不了什么，按无法复算记 + row.update(status=UNCHECKED, reason=f"{NO_DATA_SHORT}（盲算查到的那一格：{gap.get('message') or ''}）") + return values[inp["path"]] = found["value"] outcome = recompute.recompute(metric, recompute.Changes(inputs=values), recompute.Sources( load=load, outputs=outputs, masked=masks, queries=queries)) diff --git a/backend/app/engine/nodes/io.py b/backend/app/engine/nodes/io.py index c642a0fd..38ed2116 100644 --- a/backend/app/engine/nodes/io.py +++ b/backend/app/engine/nodes/io.py @@ -150,7 +150,7 @@ def _apply_contract( from datetime import datetime, timezone from app.engine.governance import board_only - from app.engine.issuance import decide_tier, incomplete_gaps, sql_check_findings, trace_numbers + from app.engine.issuance import decide_tier, incomplete_gaps, no_data_gaps, sql_check_findings, trace_numbers nodes = state.get("nodes") or {} # 写了 report_from 就是引用模式：按报告文档逐段复核，不再按数值回指叙述 @@ -191,6 +191,8 @@ def _apply_contract( gaps.append(f"指标来源未解析：{'、'.join(unresolved[:5])}") # 拿截断的查询结果整组算出来的指标：数回指得上，但算的只是取回的那一截，不能完整出具 gaps.extend(incomplete_gaps(metrics)) + # 输入取自「查询区间里这张表没有数据」的列的指标：数回指得上、复算一致，但那个 0 不代表真的是 0，不能完整出具 + gaps.extend(no_data_gaps(metrics)) # 来源查询没通过 SQL 检查（error 级：重复计算、存量跨期加总）：口径卡复算得一致，算的却是错数。 # 报告直接引用的查询要等引用核对之后才知道，缺口放在这个位置，下面一起算 sql_at = len(gaps) @@ -489,6 +491,8 @@ def _check_citations( "counted": governed and bool(unknown)} out["queries"], out["aliases"] = _cited_queries(doc, catalog) + # 报告直接引用了「查询区间里这张表没有数据」的列的格（查询快照的 no_data）：这些 0 不代表真的是 0，按缺口降档 + gaps.extend(_cited_no_data(doc, catalog)) # 老前端要用的逐个数字出处，从复核通过的数字片段拼出来 for _, unit in iter_units(doc): @@ -854,6 +858,37 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An } +def _cited_no_data(doc: dict[str, Any], catalog: dict[str, Any]) -> list[str]: + """报告直接引用的格里，落在「查询区间里这张表没有数据」的列上的（查询快照的 no_data，data/coverage.py）：按列一句缺口 + 「报告引用的 Q1「上周到访人数」：所依据的查询区间里这张表没有数据（…）」。读不出来的快照不算。""" + from app.core.artifact_store import load + from app.data.coverage import NO_DATA_SHORT, no_data_of + from app.engine.evidence import iter_units + + snapshots: dict[str, Any] = {} + out: list[str] = [] + seen: set[tuple[str, str]] = set() + for _, unit in iter_units(doc): + for seg in unit.get("segments") or []: + cite = seg.get("cite") or {} + if cite.get("kind") != "cell" or cite.get("status") != "resolved" or not cite.get("alias"): + continue + alias = str(cite["alias"]) + column = str((cite.get("locator") or {}).get("column") or "") + artifact = (catalog.get(alias) or {}).get("artifact") + if not column or not isinstance(artifact, str) or (alias, column) in seen: + continue + seen.add((alias, column)) + if artifact not in snapshots: + try: + snapshots[artifact] = load(artifact) + except (ValueError, OSError): + snapshots[artifact] = None + if (found := no_data_of(snapshots[artifact], column)) is not None: + out.append(f"报告引用的 {alias}「{column}」：{NO_DATA_SHORT}（{found.get('message') or ''}）") + return out + + def _cited_queries(doc: dict[str, Any], catalog: dict[str, Any]) -> tuple[dict[str, list[str]], dict[str, str]]: """报告直接引用的查询：({查询快照 id: [引用的写法…]}, {目录里每个查询的快照 id: 编号})。 diff --git a/backend/app/engine/nodes/metrics.py b/backend/app/engine/nodes/metrics.py index 85a9aa6d..ac1c88de 100644 --- a/backend/app/engine/nodes/metrics.py +++ b/backend/app/engine/nodes/metrics.py @@ -82,6 +82,8 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]: checked: dict[str, list[QueryProblem]] = {} #: 指标 id → 它没通过 SQL 检查的那几条来源查询。运行日志按查询归并（_emit_sql_check_logs） failed_by: dict[str, list[QueryProblem]] = {} + #: 查询快照工件 id → 快照本身（读输入的格落在哪一列「查询区间里这张表没有数据」，_no_data_inputs） + snapshots: dict[str, Any] = {} for definition in definitions: metric_id = str(definition.get("id") or "").strip() expr = str(definition.get("expression") or "").strip() @@ -159,6 +161,12 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]: "sql_check_sources": [{"artifact": artifact, "codes": list(dict.fromkeys( code for p in found for code in p.codes))} for artifact, found in failed]} failed_by[metric_id] = [p for _, found in failed for p in found] + # 输入取自「查询区间里这张表没有数据」的列（查询快照的 no_data，data/coverage.py）：值照算，标 no_data 并写明原因， + # 出具契约据此降档（issuance.no_data_gaps）。按输入的快照 id + 列认：同一条查询里有数的列不受影响。转抄的指标 + # 那张卡标了的，这一个照样标 + no_data: dict[str, Any] = {} + if lacking := _no_data_inputs(inputs, carried, snapshots): + no_data = {"no_data": True, "no_data_reason": _no_data_reason(lacking), "no_data_sources": lacking} metrics.append( { "id": metric_id, @@ -175,10 +183,15 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]: # 值是空的就是缺输入，不论 on_missing 是哪一种：fail 模式下直接取到空值的 # 指标（原式就是 vars.x.y 而 y 不存在）以前也是照常产出一个 None "status": "ok" if value is not None else "missing_input", + # 输入都齐、是计算式自己交回的空值（分母为 0 时写的 … if 分母 else None）：算不出来，不是缺输入。状态照旧是 + # missing_input（证据面板、推演、运行对比按状态认的地方不受影响），只多这一个键；有值的指标和以前一字不差 + **({"undefined": True} if value is None and not missing else {}), # 只在不完整时出现：完整的指标和以前一字不差，老运行复核时目录也对得上 **incomplete, # 只在来源查询没通过 SQL 检查时出现，同上 **sql_check, + # 只在输入取自「查询区间里这张表没有数据」的列时出现，同上 + **no_data, } ) @@ -188,11 +201,11 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]: # text 是给叙述节点 prompt 用的清单——它是叙述层唯一的数字来源 lines = [ - (f"- {m['id']}（{m['name']}）= {m['value']}{m['unit']}" if m["value"] is not None - else f"- {m['id']}（{m['name']}）= {MISSING}（缺少输入，无值）") + f"- {m['id']}（{m['name']}）= {_listed(m)}" # 不写行数：叙述节点照抄进正文就是一个没有出处的数 + ("（不完整：基于被截断的查询结果计算）" if m.get("incomplete") else "") + ("（存疑：所依据的查询未通过 SQL 检查）" if m.get("sql_check_failed") else "") + + (LISTED_NO_DATA if m.get("no_data") else "") for m in metrics ] for m in metrics: @@ -202,6 +215,12 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]: reason=m["incomplete_reason"], rows=[s.get("rows") for s in m.get("truncated_sources") or []], message=f"指标「{m['name']}」{m['incomplete_reason']}") _emit_sql_check_logs(metrics, failed_by, state, ctx) + if lacking := [m for m in metrics if m.get("no_data")]: + # 结构化字段给界面读：metrics / metric_names（受影响的指标）、reason（第一个指标的那句）、sources + ctx.emit(EventType.LOG, level="warn", code="metric_no_data", metric=lacking[0]["id"], + metrics=[m["id"] for m in lacking], metric_names=[m["name"] for m in lacking], + reason=lacking[0]["no_data_reason"], + message="指标" + "".join(f"「{m['name']}」" for m in lacking) + lacking[0]["no_data_reason"]) card = { "kind": "metric_set", "caliber": caliber, @@ -443,6 +462,72 @@ def _rendered(value: Any, definition: dict[str, Any], display: dict[str, Any]) - return MISSING +#: 清单里输入取自「查询区间里这张表没有数据」的列的指标后面那句：直接读清单的模型节点不能把 0 当成真的是 0 +LISTED_NO_DATA = "（这段时间表里没有数据：不要当成 0 解读）" +#: 指标的 no_data_reason 打头那半句（后面接查询记下的说明） +NO_DATA_REASON = "所依据的查询区间里这张表没有数据" + + +def _no_data_inputs(inputs: list[dict[str, Any]], carried: list[dict[str, Any]], + snapshots: dict[str, Any]) -> list[dict[str, Any]]: + """这个指标的输入里，取自「查询区间里这张表没有数据」的列的：[{artifact, column, table, window, message}]。 + + 认两处：输入的出处带着查询快照 id 和列（cell() 取的格、按位置取的格、Agent 经 cite_fields 交来的字段），快照里的 + no_data 记着这一列；转抄的指标那张卡标了 no_data 的（no_data_sources 照抄）。读不出来的快照不算。""" + from app.data.coverage import no_data_of + + out: list[dict[str, Any]] = [] + for item in inputs: + artifact = item.get("artifact") + locator = item.get("locator") if isinstance(item.get("locator"), dict) else {} + columns = [locator.get("column")] if locator.get("column") is not None \ + else list((locator.get("columns") or {}).values()) if isinstance(locator.get("columns"), dict) else [] + if not isinstance(artifact, str) or not artifact or not columns: + continue + if artifact not in snapshots: + try: + snapshots[artifact] = artifact_store.load(artifact) + except (ValueError, OSError): + snapshots[artifact] = None + for column in columns: + found = no_data_of(snapshots[artifact], column) + if found is None: + continue + entry = {"artifact": artifact, "column": str(column), "table": found.get("table"), + "window": found.get("window"), "message": found.get("message")} + if entry not in out: + out.append(entry) + for metric in carried: + for entry in metric.get("no_data_sources") or []: + if isinstance(entry, dict) and entry not in out: + out.append(dict(entry)) + return out + + +def _no_data_reason(lacking: list[dict[str, Any]]) -> str: + """「所依据的查询区间里这张表没有数据：「上周到访人数」查的是 … 」：出具缺口、运行日志、证据面板、写作目录都用这句。""" + said = list(dict.fromkeys(str(e.get("message") or "") for e in lacking if e.get("message"))) + return NO_DATA_REASON + (f"：{'；'.join(said[:2])}" if said else "") + + +#: 清单里没有值的两种说法：缺输入，和输入都齐、计算式自己算不出来（分母为 0） +LISTED_MISSING = "（缺少输入，无值）" +LISTED_UNDEFINED = "（算不出来：这组输入下计算式没有值，比如分母为 0）" +#: 有值、按口径卡的小数位和格式显示不出来（太小、太大、不是有限的数）：不把原值拼出来，照实说 +LISTED_UNSHOWABLE = "（有值，但按设定的小数位和格式显示不出来）" + + +def _listed(metric: dict[str, Any]) -> str: + """清单里这个指标的值：卡上按单位、小数位、格式渲染好的字（rendered，和报告里经 [[m:]] 渲染出来的一样）。以前拼原值 + （{value}{unit}），比率 0.0007 配 % 写成「0.0007%」，直接读清单的模型节点差 100 倍。""" + if metric["value"] is None: + return MISSING + (LISTED_UNDEFINED if metric.get("undefined") else LISTED_MISSING) + rendered = str(metric.get("rendered") or "") + if not rendered or rendered == MISSING: + return MISSING + LISTED_UNSHOWABLE + return rendered + + def _value_of(path: str, tctx: dict[str, Any]) -> Any: """单独取一个引用的值。取不到（或取值本身出错）一律当作没有值。""" try: diff --git a/backend/app/engine/nodes/report.py b/backend/app/engine/nodes/report.py index 00ab5795..f3efa3cf 100644 --- a/backend/app/engine/nodes/report.py +++ b/backend/app/engine/nodes/report.py @@ -56,6 +56,8 @@ from app.engine.evidence import ( CLAIMS_RULE, ENTITY_KINDS, MARKER_RULES, + NO_DATA_MARK, + NO_DATA_RULE, SEE_AND_CODE_RULES, UNIT_RULE, StreamRenderer, @@ -651,6 +653,9 @@ def _messages(ctx: NodeContext, state: GraphState, writer: WriterCatalog, claims if writer.guarded: # 写作者看不全证据（目录有省略、有来源的结构化结论为空）：「没有查到」这类结论没有依据 rules = f"{rules}\n{ABSENCE_RULE}" + if NO_DATA_MARK in writer.text: + # 有指标、查询列取自「查询区间里这张表没有数据」的列（data/coverage.py）：照实写没有数据，不写成 0、持平 + rules = f"{rules}\n{NO_DATA_RULE}" if claims in ("require_citation", CLAIMS_JUDGE): # 裁判只看写作者挂的依据：要判，就更得每句都挂上 rules = f"{rules}\n{CLAIMS_RULE}" + (f"\n{judging.JUDGE_RULE}" if claims == CLAIMS_JUDGE else "") diff --git a/backend/app/engine/recompute.py b/backend/app/engine/recompute.py index 78cd0e4b..690e6a48 100644 --- a/backend/app/engine/recompute.py +++ b/backend/app/engine/recompute.py @@ -609,7 +609,8 @@ _ANY = object() _CARD_PLAIN = frozenset({"kind", "caliber", "caliber_version", "artifact", "source"}) #: 一个指标里不带值的字段（名称、单位、表达式、显示格式、状态这类）。值、渲染、输入、代入式都带着值 _METRIC_PLAIN = frozenset({"id", "name", "unit", "expression", "decimals", "format", "status", "recompute_ok", - "incomplete", "incomplete_reason", "sql_check_failed", "sql_check_reason"}) + "incomplete", "incomplete_reason", "sql_check_failed", "sql_check_reason", "undefined", + "no_data", "no_data_reason", "no_data_sources"}) #: 转抄的上游口径卡当时的产出读不出来（文件不在、与哈希不一致、不知道是谁写的）：转抄来的是不是遮罩值核对不了 CARD_UNREADABLE = "转抄的口径卡结果无法读取，核对不了这个值是否取自设置了遮罩的列" diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index 308d06db..f2a52b73 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -137,11 +137,18 @@ WARN_CODES: dict[str, tuple[str, str]] = { # 核查 agent 的输出到了上限被截断（推理模型的思考也算在额度里）：没交成结果，由节点按无法复算、未能反查记缺口。 # 同 verify_limit，是缺口不是断裂，记成它自己的 kind：说得清是输出额度的问题，换个模型就好 "verify_truncated": ("verify_truncated", DEGRADED), + # 查询区间里这张表没有数据（data/coverage.py）：查询结果整列是 0，口径卡的指标取自这些列。答案照常交付（出具契约另按 + # 缺口降档），是缺口不是断裂；但换一张有这段数据的表再建一次大概率就好，所以能给出重来的指示（REBUILD） + "query_no_data": ("query_no_data", DEGRADED), + "metric_no_data": ("metric_no_data", DEGRADED), } # 这些 kind 重跑一次大概率能好：原因说得清，且不是「问题本身没问明白」。 # 工具参数错了、步数不够、图没校验过，都属于改一下再来一遍就行的。 RETRYABLE = {"step_limit", "tool_error", "empty_completion", "node_failed"} +#: 是缺口（DEGRADED）、却值得重建一次的信号：查询区间里这张表没有数据——图选错了表，换一张有这段数据的表大概率就好。 +#: 答案照常交付，复核照样可以给出「按复核重来」的指示（should_retry、chat/turn.retry_after_review 都认） +REBUILD = {"query_no_data", "metric_no_data"} #: 串行工具模式下塞给未执行调用的占位说明。它漏到用户面前当过答案， #: 所以复核要认得出来。从 nodes.llm 导入会绕成循环，这里按内容识别。 @@ -357,8 +364,8 @@ def worst(signals: Iterable[Signal]) -> str: def should_retry(signals: Iterable[Signal]) -> bool: - """这些信号里有没有「重跑一次大概率能好」的。""" - return any(s.kind in RETRYABLE and s.severity == BROKEN for s in signals) + """这些信号里有没有「重跑一次大概率能好」的：严重的那几类（RETRYABLE），或者值得重建一次的缺口（REBUILD）。""" + return any((s.kind in RETRYABLE and s.severity == BROKEN) or s.kind in REBUILD for s in signals) def check_numbers(original: str, revised: str) -> list[str]: diff --git a/backend/app/engine/schema.py b/backend/app/engine/schema.py index 5f37fa06..9adf1440 100644 --- a/backend/app/engine/schema.py +++ b/backend/app/engine/schema.py @@ -1464,6 +1464,40 @@ def _caliber_inputs(card: GraphNode, spec: GraphSpec) -> list[tuple[int, GraphNo return out +def _is_zero(node: Any) -> bool: + """常量 0（0、0.0、-0、+0）；布尔不算。""" + import ast + + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)): + node = node.operand + return isinstance(node, ast.Constant) and type(node.value) in (int, float) and node.value == 0 + + +def _divides(node: Any) -> bool: + import ast + + return any(isinstance(n, ast.BinOp) and isinstance(n.op, (ast.Div, ast.FloorDiv, ast.Mod)) for n in ast.walk(node)) + + +def zero_for_undefined(expression: str) -> bool: + """口径卡的表达式在算不出来时给了 0：某个条件表达式（x if c else y，嵌套的也算）一边有除法、另一边是常量 0—— + `round((a - b) / b, 4) if b else 0`、`0 if not b else a / b`。分母为 0、没有数据时给 0，报告里写成「0%」「持平」， + 读的人当成真的没变；应给 None（显示「—」）。按语法树认，解析不了的不算（另有表达式错误的报法）。""" + import ast + + from app.engine.expressions import ExpressionError, parse_expression + + try: + tree, _, _ = parse_expression(expression) + except (ExpressionError, ValueError, TypeError): + return False + for n in ast.walk(tree): + if isinstance(n, ast.IfExp) and ((_divides(n.body) and _is_zero(n.orelse)) + or (_divides(n.orelse) and _is_zero(n.body))): + return True + return False + + def evidence_issues(spec: GraphSpec) -> list[ValidationIssue]: from app.engine.governance import UPGRADE_POLICIES, UPGRADE_POLICY_LABELS @@ -1488,6 +1522,13 @@ def evidence_issues(spec: GraphSpec) -> list[ValidationIssue]: elif node.type == NodeType.METRICS: if cfg.get("caliber_from") not in (None, "", {}): out.extend(_caliber_from_issues(node, UPGRADE_POLICIES, UPGRADE_POLICY_LABELS, spec)) + else: + for i, d in enumerate(cfg.get("metrics") or []): + if isinstance(d, dict) and zero_for_undefined(str(d.get("expression") or "")): + add(f"指标「{d.get('name') or d.get('id') or i + 1}」在算不出来时给了 0（条件表达式的一边是除法，" + "另一边是 0）：分母为 0、没有数据时应给 None（报告里显示「—」，写作者照实说算不出来），" + "写成 0 会被当成持平 / 0%。请改成「… if 分母 else None」", + node, f"metrics[{i}].expression", code="metrics.zero_for_undefined") warned: set[str] = set() for i, producer in _caliber_inputs(node, spec): role = str(producer.config.get("evidence_role") or "compute") diff --git a/backend/app/tools/datasource.py b/backend/app/tools/datasource.py index fe249549..c54a6823 100644 --- a/backend/app/tools/datasource.py +++ b/backend/app/tools/datasource.py @@ -696,6 +696,12 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, # 快照和以前一字不差 if checks := await _sql_checks(_checker(), sql): payload["checks"] = checks + # 结果里有整列是 0 或空的数值列时，现查查询区间里这张表有没有数据（data/coverage.py）：没有就照实写进结果和 + # 快照（no_data，给人和模型看的 message），运行事件里记一条 query_no_data。没查出问题不加这个键，快照和以前一字不差 + if gaps := await _coverage(source, str(payload.get("sql") or sql), payload, mask): + payload["no_data"] = gaps + await _announce(ctx, EventType.LOG, level="warn", code="query_no_data", no_data=gaps, + message="；".join(g["message"] for g in gaps)) if outside and (unchecked := _unchecked_tables(sql, source, outside)): payload["unchecked_tables"] = unchecked # 表结构快照：这次查询时数据源的结构冻结下来，报告里写的表名、字段名按它核对。事后数据源 @@ -776,6 +782,33 @@ def _unchecked_tables(sql: str, source: Any, outside: frozenset[str]) -> list[st if name_key(str(t).rsplit(".", 1)[-1]) in outside and data_catalog.resolve_table_name(cache, str(t)) is None] +async def _coverage(source: Any, sql: str, payload: Mapping[str, Any], mask: list[str]) -> list[dict[str, Any]]: + """查询区间里这张表没有数据的那几列（data/coverage.result_gaps）。只在结果里有整列是 0 或空的数值列时才查库；遮罩列 + 不探查；现查表的范围不成时退回剖析、取值观察记下的范围（上传表格的快照对不上就不用）。从不抛异常。""" + from app.data import coverage + + if not coverage.zero_columns(payload): + return [] + + async def facts(table: str) -> Mapping[str, Any] | None: + from app.data.catalog_profile_jobs import last_facts + from app.db.base import SessionLocal + + source_id = getattr(source, "id", None) + if not source_id: + return None + name = data_catalog.resolve_table_name(getattr(source, "schema_cache", None), table) or table + async with SessionLocal() as session: + found = (await last_facts(session, str(source_id), [name])).get(name) + snapshot = getattr(source, "snapshot_id", None) + if found and snapshot and found.get("snapshot_id") != snapshot: + # 上传表格：剖析时是另一版数据，范围对不上这次读的这一版 + return None + return found + + return await coverage.result_gaps(source, sql, payload, facts=facts, masked=mask) + + async def _sql_checks(checker: SqlChecker, sql: str) -> list[dict[str, Any]]: """一条查询的 SQL 检查结果（as_dict 的形状）。在线程里跑、限时 SQL_CHECK_TIMEOUT_S：解析是纯计算，放在事件循环里 会挡住同一进程里别的运行。出错、超时都返回空列表——检查不能让一次成功的查询变成失败。""" diff --git a/backend/tests/test_copilot_coverage.py b/backend/tests/test_copilot_coverage.py new file mode 100644 index 00000000..0fc6624f --- /dev/null +++ b/backend/tests/test_copilot_coverage.py @@ -0,0 +1,134 @@ +"""助手搭图时就发现「查的区间里表里没有数据」（copilot_gen.GraphChecker.coverage、data/coverage.sql_gaps）。 + +部署上撞过：周报的取数节点用一张只存最新一天的表算上周、上上周，跑完才发现全是 0。现在： + +- 自查：助手搭的图里工具节点写死的 SQL，按今天跑一遍覆盖检查（不跑原 SQL，只跑算界和现查范围的两条小查询，按表去重、 + 限时）；完全不重叠的作为一条问题交回模型改一轮（换有这段数据的表，或者在说明里照实告诉用户）；修正后仍在的不挡运行， + 作为提醒留在检查结果里； +- look / db_schema 的剖析信息：日期范围最早和最晚是同一天时加一句「可能每天整表重灌：算历史周期请换有历史的表」； +- 助手提示：写按周期取数的 SQL 前先看表的日期范围。 + +库是现造的 SQLite 文件，表名、列名、数字都是虚构的。 +""" +from __future__ import annotations + +import json +import sqlite3 +import uuid + +import pytest +from langchain_core.messages import AIMessageChunk + +from app.data import introspect +from app.data.engine import engines +from app.data.schema_view import PROFILE_HINT_PROMPT, profile_lines +from app.db.base import SessionLocal +from app.db.models import DataSource +from tests.test_copilot_answer_ops import _sse, _use, client # noqa: F401 + +NO_DATA_SQL = "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012' AND event_date < '20261019'" +HAS_DATA_SQL = "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20200101'" + + +@pytest.fixture +async def plant(tmp_path): + path = tmp_path / f"{uuid.uuid4().hex[:8]}.db" + with sqlite3.connect(path) as db: + db.executescript("CREATE TABLE gate_events (event_date TEXT, member_id INTEGER);" + "INSERT INTO gate_events VALUES ('20200101', 1), ('20200101', 2);") + name = f"plant_{uuid.uuid4().hex[:6]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + await session.flush() + row.schema_cache = await introspect.introspect(row) + await session.commit() + source_id = row.id + async with SessionLocal() as session: + source = await session.get(DataSource, source_id) + yield name, source + await engines.invalidate(source_id) + + +def _node(nid: str, ntype: str, config: dict) -> dict: + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def _graph(name: str, sql: str) -> tuple[dict, list]: + nodes = {"start": _node("start", "input", {"fields": [{"name": "question"}]}), + "fetch": _node("fetch", "tool", {"tool": f"db_query__{name}", "args": {"sql": sql}}), + "out": _node("out", "output", {"fields": [{"name": "n", "value": "{{ nodes.fetch }}"}]})} + return nodes, [{"source": "start", "target": "fetch"}, {"source": "fetch", "target": "out"}] + + +async def test_the_checker_returns_it_once_then_keeps_it_as_a_warning(plant): + from app.api.copilot_gen import GraphApplier, GraphChecker + + name, source = plant + nodes, edges = _graph(name, NO_DATA_SQL) + checker = GraphChecker(GraphApplier(nodes, edges), sources=[source]) + [issue] = [i for i in await checker.issues() if i["code"] == "sql_no_data"] + assert issue["level"] == "error" and issue["node_id"] == "fetch" and issue["field"] == "args.sql" + assert "查的区间是 2026-10-12 到 2026-10-18，但 gate_events 的 event_date 只有 2020-01-01 这一天的数据" \ + in issue["message"] + assert "换有这段数据的表" in issue["for_model"] and "explanation" in issue["for_model"] + # 交回模型改过一轮：之后不再挡；收尾时作为提醒 + assert not [i for i in await checker.issues() if i["code"] == "sql_no_data"] + [warning] = await checker.coverage(level="warning") + assert warning["level"] == "warning" and warning["code"] == "sql_no_data" + + +async def test_a_window_with_data_and_templated_sql_are_left_alone(plant): + from app.api.copilot_gen import GraphApplier, GraphChecker + + name, source = plant + for sql in (HAS_DATA_SQL, "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '{{ input.day }}'"): + nodes, edges = _graph(name, sql) + checker = GraphChecker(GraphApplier(nodes, edges), sources=[source]) + assert not [i for i in await checker.issues() if i["code"] == "sql_no_data"], sql + + +async def test_the_stream_asks_for_one_repair_then_runs_with_a_warning(client, monkeypatch, plant): # noqa: F811 + name, _ = plant + nodes, edges = _graph(name, NO_DATA_SQL) + build = [{"op": "plan", "summary": "数一数"}, + *({"op": "add_node", "node": {"id": n["id"], "type": n["type"], "label": n["id"], + "config": n["data"]["config"]}} for n in nodes.values()), + *({"op": "add_edge", "edge": e} for e in edges), {"op": "done", "explanation": "数一数", "run": True}] + keep = [{"op": "done", "explanation": "gate_events 只有 2020-01-01 的数据，这段时间没有数据"}] + seen: list[str] = [] + + class Scripted: + async def astream(self, messages): + human = next(text for role, text in messages if role == "human") + seen.append(human) + for op in (keep if "上线前自查" in human else build): + yield AIMessageChunk(content=json.dumps(op, ensure_ascii=False) + "\n") + + _use(monkeypatch, Scripted()) + got = await _sse(client, {"instruction": "上周到访多少次", "intent": "answer"}) + [repairing] = [op for op in got if op["op"] == "check" and op["status"] == "repairing"] + assert [i["code"] for i in repairing["issues"]] == ["sql_no_data"] + assert any("上线前自查" in h and "换有这段数据的表" in h for h in seen), "交回模型改一轮" + [passed] = [op for op in got if op["op"] == "check" and op["status"] == "passed"] + assert [w["code"] for w in passed["warnings"]] == ["sql_no_data"] + [final] = [op for op in got if op["op"] == "final"] + assert final["next"] == "run", "修正后仍在的不挡运行" + assert any(i["code"] == "sql_no_data" and i["level"] == "warning" for i in final["issues"]) + + +def test_a_one_day_profile_says_so(): + facts = {"at": "2026-10-18T02:00:00+00:00", "date_ranges": [{"column": "event_date", "min": "20261017", + "max": "20261017"}]} + lines = profile_lines(facts, set(), None) + assert any("剖析时这张表的 event_date 只有一天（2026-10-17）的数据，可能每天整表重灌：算历史周期请换有历史的表" in ln + for ln in lines), lines + assert lines[-1].strip() == PROFILE_HINT_PROMPT + wide = {**facts, "date_ranges": [{"column": "event_date", "min": "20260101", "max": "20261017"}]} + assert not any("整表重灌" in ln for ln in profile_lines(wide, set(), None)) + + +def test_the_copilot_is_told_to_look_at_the_dates_first(): + from app.api.copilot import NODE_REFERENCE + + assert "先看表的日期范围" in NODE_REFERENCE and "只有一天的数据" in NODE_REFERENCE diff --git a/backend/tests/test_dual_check_node.py b/backend/tests/test_dual_check_node.py index f7f2d1f0..f99553b3 100644 --- a/backend/tests/test_dual_check_node.py +++ b/backend/tests/test_dual_check_node.py @@ -271,9 +271,11 @@ async def test_two_metrics_using_one_cell_are_checked_once(monkeypatch, shop): async def test_hitting_a_limit_counts_as_unchecked(monkeypatch, shop): - """查询次数用完了还没查到要交的那一格：按无法复算计入缺口；一个指标都没核成，产出是 unchecked，不是 agreed。""" + """查询次数用完了还没查到要交的那一格：按无法复算计入缺口；一个指标都没核成，产出是 unchecked，不是 agreed。 + 盲算交回的理由截一段带进原因：审批卡、缺口说明上看得到它查到了什么。""" name, _ = shop - Blind(monkeypatch, name, {"销售额": ([BLIND_TOTAL, RIGHT], "gmv")}) + Blind(monkeypatch, name, {"销售额": ([BLIND_TOTAL, RIGHT], "gmv")}, + rationale="只查到了不含退款的合计，还没来得及按支付日核对。") run = await start(graph(name, RIGHT), options={"verify": {"dual_check": {"max_queries": 1}}}) assert run.status == "succeeded", run.error evs = await events(run.id) @@ -281,15 +283,16 @@ async def test_hitting_a_limit_counts_as_unchecked(monkeypatch, shop): out = dual_output(run, evs) assert out["status"] == "unchecked" and out["metrics"][0]["status"] == "unchecked" assert out["counts"] == {"agreed": 0, "disagreed": 0, "unchecked": 1} - assert out["issuance_gaps"] == ["指标「销售额」无法复算：盲算已达每项最多查询次数（1 次）"] + reason = "盲算已达每项最多查询次数（1 次）；盲算交回的理由：只查到了不含退款的合计，还没来得及按支付日核对。" + assert out["issuance_gaps"] == [f"指标「销售额」无法复算：{reason}"] [said] = logs(evs, "dual_check_unchecked") - assert said["metrics"] == [{"id": "gmv", "name": "销售额", "reason": "盲算已达每项最多查询次数（1 次）"}] + assert said["metrics"] == [{"id": "gmv", "name": "销售额", "reason": reason}] assert not logs(evs, "dual_check_agreed") assert not [e for e in evs if e.type == "human.requested"] [finished] = [e for e in evs if e.type == "node.finished" and e.node_id == "dual"] assert finished.data["evidence"][0]["status"] == "unchecked" assert run.output["_issuance"]["tier"] == "degraded" - assert run.output["_issuance"]["gaps"] == ["双盲复算「复核销售额」：指标「销售额」无法复算：盲算已达每项最多查询次数（1 次）"] + assert run.output["_issuance"]["gaps"] == [f"双盲复算「复核销售额」：指标「销售额」无法复算：{reason}"] async def test_an_answer_on_the_last_allowed_query_still_counts(monkeypatch, shop): @@ -1575,3 +1578,41 @@ async def test_the_report_catalog_leaves_out_the_blind_queries(monkeypatch, shop if e.type == "tool.end" and e.node_id == "dual" and e.data.get("query_artifact")] ledger = [entry for e in evs if e.type == "node.finished" for entry in e.data.get("evidence") or []] assert blind_queries and not any(entry.get("artifact") in blind_queries for entry in ledger) + + +# -------------------------------------------------------------------------- +# 查询区间里这张表没有数据（data/coverage.py）：不判一致，按无法复算记 +# -------------------------------------------------------------------------- + +#: 订单表只有 2026 年的数：2030 年那一周表里没有数据，COUNT 是 0 +EMPTY_WEEK = "BETWEEN '2030-01-07' AND '2030-01-13'" +NO_ORDERS = f"SELECT COUNT(*) AS orders FROM orders WHERE status <> 'refunded' AND paid_date {EMPTY_WEEK}" +ORDERS = {"id": "orders", "name": "订单数", "unit": "单", "expression": "cell(nodes.fetch, 0, 'orders')"} + + +async def test_an_original_cell_without_data_is_not_checked(monkeypatch, shop): + """原值所在的格子落在 no_data 的列上：那个 0 不代表真的是 0，盲算查到 0 也说明不了一致。不派盲算，按无法复算记。""" + name, _ = shop + blind = Blind(monkeypatch, name, {"订单数": (NO_ORDERS, "orders")}) + run = await start(graph(name, NO_ORDERS, metrics=[ORDERS], dual={"definitions": {"orders": "已支付订单数"}})) + assert run.status == "succeeded", run.error + evs = await events(run.id) + out = dual_output(run, evs) + [row] = out["metrics"] + assert row["status"] == "unchecked" and row["reason"].startswith("查询区间里这张表没有数据（输入 cell(nodes.fetch") + assert "2030-01-07 到 2030-01-13" in row["reason"] + assert "step" not in blind.calls, "原值就没有数据：不白派一次盲算" + assert out["counts"] == {"agreed": 0, "disagreed": 0, "unchecked": 1} + + +async def test_a_blind_cell_without_data_is_not_agreed(monkeypatch, shop): + """盲算交回的格子落在 no_data 的列上（它查了一段表里没有数据的区间）：不判一致，按无法复算记。""" + name, _ = shop + buyers = {"id": "orders", "name": "订单数", "unit": "单", "expression": "cell(nodes.fetch, 0, 'orders')"} + original = f"SELECT COUNT(*) AS orders FROM orders WHERE status <> 'refunded' AND paid_date {WEEK}" + Blind(monkeypatch, name, {"订单数": (NO_ORDERS.replace("AS orders", "AS n"), "n")}) + run = await start(graph(name, original, metrics=[buyers], dual={"definitions": {"orders": "已支付订单数"}})) + assert run.status == "succeeded", run.error + [row] = dual_output(run, await events(run.id))["metrics"] + assert row["status"] == "unchecked", row + assert row["reason"].startswith("查询区间里这张表没有数据（盲算查到的那一格：「n」查的是 2030-01-07 到 2030-01-13") diff --git a/backend/tests/test_metric_lineage.py b/backend/tests/test_metric_lineage.py index 2ae24905..6338bd3c 100644 --- a/backend/tests/test_metric_lineage.py +++ b/backend/tests/test_metric_lineage.py @@ -179,7 +179,8 @@ async def test_each_metric_records_its_inputs_substitution_and_rendering(engine_ # 旧字段一个不少，text 照旧——模板 ⑧ 的叙述 prompt 靠它 assert {"id", "name", "unit", "value", "expression"} <= set(wow) output = row.output["清单"] - assert "- wow（环比增幅）= 8.7%" in output and "- orders（订单数）= 1234单" in output + # 清单按卡上渲染好的字写（和报告里 [[m:]] 一样按单位、小数位、千分位）：叙述节点抄的就是报告上会出现的写法 + assert "- wow（环比增幅）= 8.7%" in output and "- orders（订单数）= 1,234单" in output async def test_metric_set_artifact_is_in_the_ledger_and_node_finished(engine_up): diff --git a/backend/tests/test_metric_no_data.py b/backend/tests/test_metric_no_data.py new file mode 100644 index 00000000..b43dd375 --- /dev/null +++ b/backend/tests/test_metric_no_data.py @@ -0,0 +1,197 @@ +"""口径卡的输入取自「查询区间里这张表没有数据」的列时，指标照实标出来（no_data），出具按缺口降档，复核给出重来的指示。 + +部署上撞过：一张周报的到访事件表只存最新一天，6 列全是 0，口径卡里 10 个指标是 0 或 0%，周报照写「上周到访人数为 0 人」。 +查询工具现在会在结果和快照里记 no_data（data/coverage.py）；这里守它往下走的几步： + +- 口径卡按输入的快照 id + 列认（按列：同一条查询里有数的列不受影响），给指标标 no_data、写明原因；清单文字注明； + 运行日志一条 metric_no_data； +- 有出具契约的成果按缺口降档； +- 复核把 metric_no_data / query_no_data 记成缺口（DEGRADED），并给出「按复核重来」的指示：换一张有这段数据的表再建一次， + 大概率就好。 + +库是现造的 SQLite 文件，表名、列名、数字都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import sqlite3 +import uuid + +import pytest +from sqlalchemy import select + +from app.core import artifact_store +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource, Run, RunEvent +from app.engine.runner import run_manager + +WINDOW = "{c} >= strftime('%Y%m%d', 'now', '-{a} days') AND {c} < strftime('%Y%m%d', 'now', '-{b} days')" +KPI_SQL = ("SELECT (SELECT SUM(pv) FROM daily_pv WHERE day >= date('now', '-14 days')) AS 客流PV, " + "(SELECT COUNT(DISTINCT member_id) FROM gate_events WHERE " + WINDOW.format(c="event_date", a=14, b=7) + + ") AS 上周到访人数, (SELECT COUNT(DISTINCT member_id) FROM gate_events WHERE " + + WINDOW.format(c="event_date", a=21, b=14) + ") AS 上上周到访人数") +WOW = ("round((cell(nodes.q_kpi, 0, '上周到访人数') - cell(nodes.q_kpi, 0, '上上周到访人数')) " + "/ cell(nodes.q_kpi, 0, '上上周到访人数'), 4) if cell(nodes.q_kpi, 0, '上上周到访人数') else None") + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def plant(tmp_path): + path = tmp_path / f"{uuid.uuid4().hex[:8]}.db" + with sqlite3.connect(path) as db: + db.executescript("CREATE TABLE gate_events (event_date TEXT, member_id INTEGER);" + "INSERT INTO gate_events VALUES ('20200101', 1), ('20200101', 2);" + "CREATE TABLE daily_pv (day TEXT, pv INTEGER);") + for back in range(30): + db.execute("INSERT INTO daily_pv VALUES (date('now', ?), 100)", (f"-{back} days",)) + name = f"plant_{uuid.uuid4().hex[:6]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + await session.commit() + source_id = row.id + yield name + await engines.invalidate(source_id) + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def board(source: str) -> dict: + nodes = [ + node("start", "input", "开始"), + node("q_kpi", "tool", "取数", tool=f"db_query__{source}", args={"sql": KPI_SQL}), + node("card", "metrics", "周报口径卡", caliber="周报口径", caliber_version="v1", metrics=[ + {"id": "pv", "name": "客流PV", "unit": "次", "expression": "cell(nodes.q_kpi, 0, '客流PV')"}, + {"id": "visitors", "name": "上周到访人数", "unit": "人", "expression": "cell(nodes.q_kpi, 0, '上周到访人数')"}, + {"id": "wow", "name": "到访周环比", "decimals": 4, "format": "percent_of_ratio", "expression": WOW}, + ]), + node("out", "output", "成果", fields=[{"name": "指标", "value": "{{ nodes.card.text }}"}], + contract={"metrics_from": ["card"], "required": ["pv", "visitors"]}), + ] + return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +async def finished(graph: dict) -> tuple[Run, list[RunEvent]]: + run = await run_manager.start(graph=graph, input_payload={}) + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + assert row.status == "succeeded", row.error + async with SessionLocal() as session: + events = list((await session.execute( + select(RunEvent).where(RunEvent.run_id == run.id).order_by(RunEvent.seq))).scalars()) + return row, events + raise AssertionError(f"运行 {run.id} 没跑完：{row.status}") + + +def _card(events: list[RunEvent]) -> dict: + [event] = [e for e in events if e.type == "node.finished" and e.node_id == "card"] + [entry] = event.data["evidence"] + return artifact_store.load(entry["artifact"]) + + +async def test_metrics_fed_by_columns_without_data_are_flagged(plant): + row, events = await finished(board(plant)) + metrics = {m["id"]: m for m in _card(events)["metrics"]} + visitors, wow, pv = metrics["visitors"], metrics["wow"], metrics["pv"] + assert visitors["value"] == 0 and visitors["no_data"] is True + assert visitors["no_data_reason"].startswith("所依据的查询区间里这张表没有数据：") + assert "gate_events" in visitors["no_data_reason"] + [source] = visitors["no_data_sources"] + assert source["column"] == "上周到访人数" and source["table"] == "gate_events" and source["artifact"] + # 周环比两个输入都在没有数据的列上：照样标；它本身算不出来（分母为 0），两件事都照实记 + assert wow["no_data"] is True and wow["undefined"] is True and wow["value"] is None + # 同一条查询里有数的列（客流 PV）不受影响 + assert "no_data" not in pv and pv["value"] > 0 + lines = row.output["指标"].split("\n") + assert "- visitors（上周到访人数）= 0人（这段时间表里没有数据：不要当成 0 解读）" in lines + assert not any("pv（客流PV）" in ln and "没有数据" in ln for ln in lines) + # 运行日志：查询一条 query_no_data，口径卡一条 metric_no_data（受影响的指标都列出来） + codes = [(e.data or {}).get("code") for e in events if e.type == "log"] + assert "query_no_data" in codes + [logged] = [e.data for e in events if e.type == "log" and (e.data or {}).get("code") == "metric_no_data"] + assert logged["level"] == "warn" and logged["metrics"] == ["visitors", "wow"] + + +async def test_the_contract_degrades_on_a_metric_without_data(plant): + row, _ = await finished(board(plant)) + issuance = row.output["_issuance"] + assert issuance["tier"] == "degraded" + gaps = [g for g in issuance["gaps"] if "没有数据" in g] + assert any("上周到访人数" in g for g in gaps) and any("到访周环比" in g for g in gaps), issuance["gaps"] + + +async def test_review_marks_it_and_asks_for_a_rebuild(plant): + from app.chat import turn + from app.engine import review as rv + + row, events = await finished(board(plant)) + signals = rv.scan(events, row.output) + kinds = {s.kind: s.severity for s in signals} + assert kinds.get("metric_no_data") == rv.DEGRADED and kinds.get("query_no_data") == rv.DEGRADED + assert rv.should_retry(signals), "换一张有这段数据的表再建一次大概率就好" + result = rv.ReviewResult(verdict="annotated", note="…", retry=True, signals=signals, severity=rv.worst(signals)) + assert turn.retry_after_review(result.as_dict(), 0) and not turn.retry_after_review(result.as_dict(), 1) + # 别的降级信号照旧不重来 + other = [rv.Signal(kind="node_retry", detail="重试过一次", severity=rv.DEGRADED)] + assert not rv.should_retry(other) + assert not turn.retry_after_review(rv.ReviewResult(verdict="annotated", retry=True, signals=other, + severity=rv.DEGRADED).as_dict(), 0) + + +async def test_the_writer_is_told_and_given_the_rule(plant): + """写作目录里这样的指标、查询格子里这样的列都写清楚；写作规则加一句。""" + from app.engine.evidence import NO_DATA_RULE, build_catalog, writer_catalog + + row, events = await finished(board(plant)) + card = _card(events) + [query] = [e for e in events if e.type == "node.finished" and e.node_id == "q_kpi"] + ledger = [*query.data["evidence"], *[e for e in events if e.type == "node.finished" + and e.node_id == "card"][0].data["evidence"]] + catalog = build_catalog(nodes={"card": {**card, "artifact": ledger[-1]["artifact"]}}, ledger=ledger, inputs={}) + assert catalog["m:visitors"]["no_data"] is True and "no_data" not in catalog["m:pv"] + text = writer_catalog(catalog).text + visitors = next(ln for ln in text.splitlines() if "[[m:visitors]]" in ln) + assert "这段时间表里没有数据" in visitors and "不要当成 0 解读" in visitors + assert "这段时间表里没有数据" not in next(ln for ln in text.splitlines() if "[[m:pv]]" in ln) + assert "（注意：「上周到访人数」" in text and "引用这几列时照实写这段时间表里没有数据" in text + assert "照实写这段时间表里没有数据" in NO_DATA_RULE and "0%" in NO_DATA_RULE + + +async def test_the_report_prompt_carries_the_rule(plant, monkeypatch): + from app.engine.evidence import NO_DATA_RULE + from tests.test_report_node import script + + seen = script(monkeypatch, "上周到访人数这段时间表里没有数据。") + graph = board(plant) + out = graph["nodes"].pop() + graph["nodes"] += [node("write", "report", "撰写", instructions="写一段周报"), + {**out, "data": {**out["data"], "config": {"fields": [{"name": "答案", + "value": "{{ nodes.write.text }}"}]}}}] + graph["edges"] = [{"source": a["id"], "target": b["id"]} for a, b in zip(graph["nodes"], graph["nodes"][1:])] + await finished(graph) + prompt = "\n".join(str(m.content) for m in seen[0]) + assert NO_DATA_RULE in prompt and "这段时间表里没有数据" in prompt + + +def test_an_ungated_result_says_its_gaps_on_the_stamp(): + """成果没配出具契约时，实例页的印章照实写这些缺口（配了契约的已经按缺口降档）。""" + from app.apps.resolve import ungated_notes + + cards = [{"metrics": [{"id": "visitors", "name": "上周到访人数", "no_data": True}, {"id": "pv", "name": "客流PV"}]}] + dual = {"issuance_gaps": ["指标「销售额」无法复算：盲算已达每项最多查询次数（1 次）"]} + notes = ungated_notes(cards, dual) + assert notes[0].startswith("指标「上周到访人数」所依据的查询区间里表里没有数据") and "没有配出具契约" in notes[0] + assert notes[1].startswith("双盲复算有缺口：指标「销售额」无法复算") + assert ungated_notes([{"metrics": [{"id": "pv", "name": "客流PV"}]}], None) == [] diff --git a/backend/tests/test_metrics_undefined.py b/backend/tests/test_metrics_undefined.py new file mode 100644 index 00000000..2b5500c7 --- /dev/null +++ b/backend/tests/test_metrics_undefined.py @@ -0,0 +1,183 @@ +"""口径卡：分母为 0 时算不出来就是空值，和「缺少输入」分开说；指标清单按格式渲染。 + +部署上撞过：一份周报的口径卡全写成 `round((a - b) / b, 4) if b else 0`，两周都没有数据时周环比、复访率写成 0 / 0%， +读起来像「持平」。现在： + +- 助手的节点手册写明分母可能为 0 时写 `… if 分母 else None`，不要写 else 0；画布校验对「条件表达式一边有除法、另一边是 + 常量 0」给提醒（metrics.zero_for_undefined，按语法树认，嵌套的也认）； +- 输入都齐、是计算式自己交回空值的指标多记 undefined: true（状态仍是 missing_input，老运行、按状态认的地方不受影响）， + 清单、写作目录、引用解析不了的原因、运行对比都说「算不出来」，不说「缺少输入」； +- 指标清单（{{ nodes.卡.text }}）按卡上渲染好的字写：比率 0.0007 显示成 0.07%，不再拼原值写成「0.0007%」；值不为空 + 却显示不出来的照实说。 + +数据、口径都是虚构的。 +""" +from __future__ import annotations + +import asyncio + +import pytest +from sqlalchemy import select + +from app.core import artifact_store +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent +from app.engine.runner import run_manager +from app.engine.schema import GraphSpec, validate_graph + +KPI = "{'cur': 0, 'prev': 0, 'visits': 1200, 'repeat': 9, 'gmv': 45678.5, 'tiny': 0.000000001}" +METRICS = [ + {"id": "wow", "name": "周环比", "decimals": 4, "format": "percent_of_ratio", + "expression": "round((vars.kpi.cur - vars.kpi.prev) / vars.kpi.prev, 4) if vars.kpi.prev else None"}, + {"id": "lost", "name": "缺一项", "expression": "vars.kpi.nothing"}, + {"id": "rate", "name": "复访率", "unit": "%", "decimals": 4, "format": "percent_of_ratio", + "expression": "round(vars.kpi.repeat / vars.kpi.visits, 4)"}, + {"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}, + {"id": "tiny", "name": "很小的数", "decimals": 2, "expression": "vars.kpi.tiny"}, +] + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def graph(metrics=METRICS, kpi=KPI): + nodes = [node("start", "input"), node("fetch", "transform", mode="expression", expression=kpi, assign_to="kpi"), + node("card", "metrics", caliber="周报口径", caliber_version="v1", metrics=metrics), + node("out", "output", fields=[{"name": "清单", "value": "{{ nodes.card.text }}"}])] + return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +async def _card() -> tuple[Run, dict]: + run = await run_manager.start(graph=graph(), input_payload={}) + for _ in range(300): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + break + assert row.status == "succeeded", row.error + async with SessionLocal() as session: + [event] = (await session.execute(select(RunEvent).where( + RunEvent.run_id == run.id, RunEvent.type == "node.finished", RunEvent.node_id == "card"))).scalars() + [entry] = event.data["evidence"] + return row, artifact_store.load(entry["artifact"]) + + +async def test_undefined_is_told_apart_from_a_missing_input(): + row, card = await _card() + by_id = {m["id"]: m for m in card["metrics"]} + wow, lost = by_id["wow"], by_id["lost"] + # 状态都是 missing_input（按状态认的地方照旧）；算不出来的多一个 undefined + assert wow["value"] is None and wow["status"] == "missing_input" and wow["undefined"] is True + assert lost["value"] is None and lost["status"] == "missing_input" and "undefined" not in lost + # 有值的指标没有这个键：卡和以前一字不差 + assert all("undefined" not in by_id[k] for k in ("rate", "gmv", "tiny")) + lines = row.output["清单"].split("\n") + assert "- wow（周环比）= —（算不出来：这组输入下计算式没有值，比如分母为 0）" in lines + assert "- lost（缺一项）= —（缺少输入，无值）" in lines + + +async def test_the_list_shows_the_rendered_value(): + row, _ = await _card() + lines = row.output["清单"].split("\n") + assert "- rate（复访率）= 0.75%" in lines, lines # 不是「0.0075%」 + assert "- gmv（销售额）= 45,678.5元" in lines + # 卡先按小数位取整，再按格式渲染：很小的数取整成 0 就显示 0.00（和报告里一样），不拼原值 + assert "- tiny（很小的数）= 0.00" in lines and "0.000000001" not in row.output["清单"] + + +def test_a_value_that_cannot_be_shown_is_said_so(): + """有值、按口径卡的格式显示不出来（不是有限的数这类，卡上 rendered 记成「—」）：照实说，不把原值拼出来。""" + from app.engine.nodes.metrics import _listed + + assert _listed({"value": float("inf"), "rendered": "—"}) == "—（有值，但按设定的小数位和格式显示不出来）" + assert _listed({"value": None, "rendered": "—", "undefined": True}).startswith("—（算不出来") + + +# -------------------------------------------------------------------------- +# 画布校验：「分母为 0 时给 0」 +# -------------------------------------------------------------------------- + + +def _zero_warnings(expression: str) -> list: + g = graph(metrics=[{"id": "x", "name": "x", "expression": expression}]) + issues = validate_graph(GraphSpec.model_validate(g)).issues + return [i for i in issues if i.code == "metrics.zero_for_undefined"] + + +@pytest.mark.parametrize("expression", [ + "round((vars.kpi.cur - vars.kpi.prev) / vars.kpi.prev, 4) if vars.kpi.prev else 0", + "vars.kpi.repeat / vars.kpi.visits if vars.kpi.visits else 0.0", + "0 if not vars.kpi.visits else vars.kpi.repeat / vars.kpi.visits", # 两边对调 + "round(vars.kpi.repeat / vars.kpi.visits if vars.kpi.visits > 0 else 0, 4)", # 套在 round 里 + "vars.kpi.gmv if vars.kpi.cur else (vars.kpi.repeat / vars.kpi.visits if vars.kpi.visits else 0)", # 嵌套 +]) +def test_zero_for_undefined_is_warned(expression): + [issue] = _zero_warnings(expression) + assert issue.level == "warning" and issue.field == "metrics[0].expression" + assert "None" in issue.message and "持平" in issue.message + + +@pytest.mark.parametrize("expression", [ + "round((vars.kpi.cur - vars.kpi.prev) / vars.kpi.prev, 4) if vars.kpi.prev else None", + "vars.kpi.cur if vars.kpi.prev else 0", # 没有除法：0 可能就是想要的值 + "vars.kpi.repeat / vars.kpi.visits", + "vars.kpi.repeat / vars.kpi.visits if vars.kpi.visits else 1", +]) +def test_other_expressions_are_left_alone(expression): + assert _zero_warnings(expression) == [] + + +def test_the_copilot_is_told_to_write_none(): + from app.api.copilot import NODE_REFERENCE + + start = NODE_REFERENCE.index("- metrics：口径卡") + section = NODE_REFERENCE[start:NODE_REFERENCE.index("\n- dual_check：", start)] + assert "if vars.kpi.gmv_prev else None" in section and "不要写 else 0" in section + + +# -------------------------------------------------------------------------- +# 写作目录、引用、对比的说法 +# -------------------------------------------------------------------------- + + +def _catalog(undefined: bool): + from app.engine.evidence import build_catalog + + card = {"kind": "metric_set", "caliber": "周报口径", "caliber_version": "v1", "artifact": "c" * 64, "metrics": [ + {"id": "wow", "name": "周环比", "unit": "", "value": None, "decimals": 4, "format": "percent_of_ratio", + "status": "missing_input", **({"undefined": True} if undefined else {}), "expression": "…"}]} + ledger = [{"kind": "metric_set", "node_id": "card", "exec": 1, "artifact": "c" * 64, "caliber": "周报口径", + "version": "v1", "metrics": ["wow"]}] + return build_catalog(nodes={"card": card}, ledger=ledger, inputs={}) + + +@pytest.mark.parametrize("undefined, said", [(True, "算不出来"), (False, "缺少输入")]) +def test_the_writer_and_the_reason_say_which(undefined, said): + from app.engine.evidence import compose_doc, writer_catalog + + catalog = _catalog(undefined) + assert catalog["m:wow"].get("undefined", False) is undefined + text = writer_catalog(catalog).text + line = next(ln for ln in text.splitlines() if "[[m:wow]]" in ln) + assert ("算不出来" in line) is undefined and "不要引用" in line + doc = compose_doc("周环比 [[m:wow]]。", catalog) + [violation] = [v for v in doc["violations"] if v["code"] == "unresolved_ref"] + assert said in violation["message"], violation + + +def test_compare_says_which(): + from app.engine import compare + + assert compare.missing_reason("这次运行", {"status": "missing_input", "undefined": True}) \ + == "这次运行这个指标算不出来（这组输入下计算式没有值，比如分母为 0）" + assert compare.missing_reason("基准运行", {"status": "missing_input"}) == "基准运行这个指标缺少输入，没有值" + assert compare.missing_reason("基准运行", {"status": "ok"}) is None diff --git a/backend/tests/test_query_coverage.py b/backend/tests/test_query_coverage.py new file mode 100644 index 00000000..f8fcfe54 --- /dev/null +++ b/backend/tests/test_query_coverage.py @@ -0,0 +1,288 @@ +"""查询区间里这张表没有数据时，查询结果里照实说（data/coverage.py，接在 tools/datasource 的查询工具上）。 + +部署上撞过：一张到访事件表只存最新一天（每天整表重灌），周报按运行当天现算的区间取上周、上上周的到访人数，6 列全是 0， +报告照写「上周到访人数为 0 人」。剖析记过日期范围，但对滚动表不可靠，所以运行时现查： + +- 结果里有整列是 0 或空的数值列时，把 SQL 收窄到这一列，认出它用到的表和时间谓词；界是字面量的直接解析，按运行当天现算 + 的在同一个库里算出来；这一列在表里的范围现查 MIN / MAX，查不成退回剖析记下的范围（照实标「按 X 日的剖析」）； +- 区间和范围完全不重叠才算没有数据：结果和快照里多一项 no_data（给人和模型看的 message），运行事件里一条 query_no_data； + 没查出问题不加这个键，快照和以前一字不差； +- 有数的列、一行都没有的结果、遮罩列、写法对不上（文本列 '20261012' 对文本界 '2026-10-12'）、认不出的一律不判。 + +库是现造的 SQLite 文件，表名、列名、数字都是虚构的。 +""" +from __future__ import annotations + +import json +import sqlite3 +import uuid + +import pytest + +from app.core import artifact_store +from app.data import coverage, introspect +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource +from app.tools.datasource import build_datasource_tools +from app.tools.registry import ToolContext, Versions + +#: 这张表只存一天（2020-01-01），按今天现算的上周一定落在它外面 +ONLY_DAY = "20200101" +#: 按今天现算的「上周」（SQLite 的写法，和列一样写成 'YYYYMMDD'） +WINDOW = "{c} >= strftime('%Y%m%d', 'now', '-14 days') AND {c} < strftime('%Y%m%d', 'now', '-7 days')" +KPI_SQL = ("SELECT (SELECT COUNT(DISTINCT v.member_id) FROM gate_events v WHERE " + WINDOW.format(c="v.event_date") + + ") AS 上周到访人数, (SELECT COUNT(*) FROM gate_events v WHERE " + WINDOW.format(c="v.event_date") + + ") AS 上周到访人次, (SELECT SUM(pv) FROM daily_pv WHERE day >= date('now', '-14 days')) AS 客流PV") + + +@pytest.fixture +async def plant(tmp_path): + """gate_events 只有 ONLY_DAY 一天；daily_pv 覆盖最近 30 天、每天 100。""" + path = tmp_path / f"{uuid.uuid4().hex[:8]}.db" + db = sqlite3.connect(path) + db.executescript( + "CREATE TABLE gate_events (event_date TEXT, member_id INTEGER, gate TEXT);" + f"INSERT INTO gate_events VALUES ('{ONLY_DAY}', 1, 'A'), ('{ONLY_DAY}', 2, 'B');" + "CREATE TABLE daily_pv (day TEXT, pv INTEGER);" + "CREATE TABLE iso_events (paid_on TEXT, amount INTEGER);" + "INSERT INTO iso_events VALUES ('2020-01-01', 5);") + for back in range(30): + db.execute("INSERT INTO daily_pv VALUES (date('now', ?), 100)", (f"-{back} days",)) + db.commit() + db.close() + name = f"plant_{uuid.uuid4().hex[:8]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + await session.flush() + row.schema_cache = await introspect.introspect(row) + await session.commit() + source_id = row.id + yield name, source_id + await engines.invalidate(source_id) + + +async def _query(name: str, sql: str, events: list | None = None) -> dict: + async def emit(event_type, **data): + if events is not None: + events.append((str(event_type), data)) + + async with SessionLocal() as session: + built = await build_datasource_tools([f"db_query__{name}"], ToolContext( + data_versions=Versions.CURRENT, run_id="run-coverage", node_id="q_kpi", emit=emit), session) + got = await built[0].ainvoke({"sql": sql}) + return json.loads(got) + + +async def test_zero_columns_over_a_window_without_data_are_flagged(plant): + name, _ = plant + events: list = [] + result = await _query(name, KPI_SQL, events) + assert result["rows"][0][:2] == [0, 0] + [gap] = result["no_data"] + assert gap["columns"] == ["上周到访人数", "上周到访人次"] and gap["table"] == "gate_events" + assert gap["column"] == "event_date" and gap["data_range"] == ["2020-01-01", "2020-01-01"] and gap["basis"] == "live" + start, end = gap["window"] + assert start < end and start > "2020-01-01" + assert gap["message"].startswith("「上周到访人数」等 2 列查的是 ") and "只有 2020-01-01 这一天的数据" in gap["message"] + assert "这些 0 不代表这段时间真的是 0，是这段时间表里没有数据" in gap["message"] + # 有数的列（客流 PV）不受影响 + assert all("客流PV" not in g["columns"] for g in result["no_data"]) + # 快照里也有；运行事件里有一条 query_no_data + snapshot = artifact_store.load(result["artifact"]) + assert snapshot["no_data"] == result["no_data"] + [(etype, data)] = [e for e in events if e[1].get("code") == "query_no_data"] + assert data["level"] == "warn" and data["no_data"] == result["no_data"] + + +async def test_literal_windows_are_parsed_directly(plant): + name, _ = plant + sql = ("SELECT COUNT(*) AS 到访人次 FROM gate_events WHERE event_date >= '20261012' AND event_date < '20261019'") + [gap] = (await _query(name, sql))["no_data"] + assert gap["window"] == ["2026-10-12", "2026-10-18"] # < 不带时刻：截到前一天 + + +async def test_nothing_is_added_when_the_window_has_data_or_the_column_is_not_zero(plant): + name, _ = plant + for sql in ( + f"SELECT COUNT(*) AS n FROM gate_events WHERE event_date = '{ONLY_DAY}'", # 有数 + "SELECT SUM(pv) AS pv FROM daily_pv WHERE day >= date('now', '-7 days')", # 有数 + "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20200101'", # 区间覆盖了那一天 + "SELECT gate, COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012' GROUP BY gate", # 一行都没有 + ): + result = await _query(name, sql) + assert "no_data" not in result, (sql, result.get("no_data")) + assert "no_data" not in artifact_store.load(result["artifact"]), "没查出问题：快照和以前一字不差" + + +async def test_a_text_column_against_a_differently_written_text_bound_is_not_judged(plant): + """列里存的是 '20200101'，界写的是 '2026-10-12'：库按字符串比，和按日期比对不上，不判。""" + name, _ = plant + result = await _query(name, "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '2026-10-12'") + assert "no_data" not in result + # ISO 写法对 ISO 写法：照常判 + result = await _query(name, "SELECT COUNT(*) AS n FROM iso_events WHERE paid_on >= date('now', '-7 days')") + [gap] = result["no_data"] + assert gap["column"] == "paid_on" and gap["data_range"] == ["2020-01-01", "2020-01-01"] + + +async def test_masked_columns_are_not_probed(plant): + name, source_id = plant + async with SessionLocal() as session: + row = await session.get(DataSource, source_id) + row.options = {**(row.options or {}), "mask_columns": ["event_date"]} + await session.commit() + result = await _query(name, "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012'") + assert "no_data" not in result + + +async def test_a_failed_probe_falls_back_to_the_profile(plant, monkeypatch): + """现查不成（超时、出错）：退回剖析记下的范围，照实标「按 X 日的剖析」。""" + name, _ = plant + real = coverage._probe + + async def probe(source, sql): + return None if "MIN(" in sql else await real(source, sql) + + async def facts(table): + assert table == "gate_events" + return {"at": "2026-10-01T03:00:00+00:00", "date_ranges": [{"column": "event_date", "min": "20260930", + "max": "20260930"}]} + + monkeypatch.setattr(coverage, "_probe", probe) + async with SessionLocal() as session: + source = await session.get(DataSource, (await _source_id(name))) + payload = {"columns": ["n"], "rows": [[0]], "row_count": 1, "column_types": {"n": "number"}} + [gap] = await coverage.result_gaps(source, "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012'", + payload, facts=facts) + assert gap["basis"] == "profile" and gap["basis_at"] == "2026-10-01" + assert gap["data_range"] == ["2026-09-30", "2026-09-30"] and "（现查表的范围没有成功，按 2026-10-01 的剖析）" in gap["message"] + # 剖析也没有：不判 + assert await coverage.result_gaps(source, "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012'", + payload, facts=None) == [] + + +async def _source_id(name: str) -> str: + from sqlalchemy import select + + async with SessionLocal() as session: + return (await session.execute(select(DataSource.id).where(DataSource.name == name))).scalar_one() + + +@pytest.mark.parametrize("value, kind, day, shape", [ + ("2026-10-12", None, "2026-10-12", "iso"), + ("2026-10-12T08:30:00", "datetime", "2026-10-12", "date"), + ("20261012", None, "2026-10-12", "compact"), + (20261012, None, "2026-10-12", "number"), + ("2026-13-01", None, None, None), + ("2026/10/12", None, None, None), + ("上周", None, None, None), +]) +def test_dates_are_normalised(value, kind, day, shape): + got = coverage._as_day(value, kind) + assert (got.day.isoformat() if got else None) == day and (got.shape if got else None) == shape + + +def test_zero_columns(): + payload = {"columns": ["a", "b", "c", "d"], "column_types": {"a": "number", "b": "number", "d": "text"}, + "rows": [[0, 1, None, "0"], ["0.00", 0, None, "0"]]} + assert coverage.zero_columns(payload) == ["a", "c"] + assert coverage.zero_columns({**payload, "rows": []}) == [] + + +async def test_the_weekly_report_shape_on_mysql(monkeypatch): + """部署上那张周报的形状（MySQL）：一条查询用标量子查询一次取几列，上周、上上周按运行当天现算；到访表只有最新一天， + 列存成 VARCHAR 'YYYYMMDD'。界由库算出来是 DATE 类型：MySQL 把文本列转成日期再比，照常判。两段区间各记一条。""" + from types import SimpleNamespace + + last = ("event_date >= DATE_SUB(CURRENT_DATE, INTERVAL (WEEKDAY(CURRENT_DATE) + 7) DAY) " + "AND event_date < DATE_SUB(CURRENT_DATE, INTERVAL WEEKDAY(CURRENT_DATE) DAY)") + before = ("event_date >= DATE_SUB(CURRENT_DATE, INTERVAL (WEEKDAY(CURRENT_DATE) + 14) DAY) " + "AND event_date < DATE_SUB(CURRENT_DATE, INTERVAL (WEEKDAY(CURRENT_DATE) + 7) DAY)") + sql = (f"SELECT (SELECT SUM(pv) FROM pv_daily WHERE stat_day >= '2026-10-01') AS `客流PV`, " + f"(SELECT COUNT(DISTINCT member_id) FROM gate_events WHERE {last}) AS `上周到访人数`, " + f"(SELECT COUNT(*) FROM gate_events WHERE {last}) AS `上周到访人次`, " + f"(SELECT COUNT(DISTINCT member_id) FROM gate_events WHERE {before}) AS `上上周到访人数`") + days = {"(WEEKDAY(CURRENT_DATE) + 14)": "2026-09-28", "(WEEKDAY(CURRENT_DATE) + 7)": "2026-10-05", + "(WEEKDAY(CURRENT_DATE)) DAY": "2026-10-12"} # sqlglot 写回 MySQL 时给这一项加了括号 + seen: list[str] = [] + + async def probe(source, text): + seen.append(text) + if "MIN(" in text: + return ["20261019", "20261019"], ["text", "text"] + values = [] + for part in text[len("SELECT "):].split(" AS b")[:-1]: + values.append(next(v for k, v in days.items() if k in part)) + return values, ["date"] * len(values) + + monkeypatch.setattr(coverage, "_probe", probe) + source = SimpleNamespace(name="plant", kind="mysql", schema_cache=None, id=None) + payload = {"columns": ["客流PV", "上周到访人数", "上周到访人次", "上上周到访人数"], + "column_types": {"客流PV": "number", "上周到访人数": "number", "上周到访人次": "number", + "上上周到访人数": "number"}, + "rows": [[3200, 0, 0, 0]], "row_count": 1} + gaps = await coverage.result_gaps(source, sql, payload) + by_window = {tuple(g["window"]): g for g in gaps} + assert set(by_window) == {("2026-10-05", "2026-10-11"), ("2026-09-28", "2026-10-04")} + assert by_window[("2026-10-05", "2026-10-11")]["columns"] == ["上周到访人数", "上周到访人次"] + assert by_window[("2026-09-28", "2026-10-04")]["columns"] == ["上上周到访人数"] + assert all(g["data_range"] == ["2026-10-19", "2026-10-19"] for g in gaps) + # 表的范围同一张表、同一列只查一次；按运行当天现算的界一条 SELECT 算出来 + assert sum("MIN(" in t for t in seen) == 1 + assert all(t.startswith("SELECT ") for t in seen) + + +async def test_a_live_range_is_kept_for_a_while(monkeypatch): + """现查到的范围在进程里记几分钟：一张图里几个取数节点、助手自查几轮查同一张表，不再每次全表 MIN / MAX。""" + from types import SimpleNamespace + + seen: list[str] = [] + + async def probe(source, text): + seen.append(text) + return ["20200101", "20200101"], ["text", "text"] + + monkeypatch.setattr(coverage, "_probe", probe) + monkeypatch.setattr(coverage, "_LIVE", {}) + source = SimpleNamespace(name=f"plant_{uuid.uuid4().hex[:6]}", kind="sqlite", schema_cache=None, id="s1") + payload = {"columns": ["n"], "rows": [[0]], "row_count": 1, "column_types": {"n": "number"}} + sql = "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012'" + for _ in range(2): + assert len(await coverage.result_gaps(source, sql, payload)) == 1 + assert sum("MIN(" in t for t in seen) == 1 + monkeypatch.setattr(coverage, "LIVE_TTL_S", 0.0) + coverage._LIVE.clear() + await coverage.result_gaps(source, sql, payload) + await coverage.result_gaps(source, sql, payload) + assert sum("MIN(" in t for t in seen) == 3, "过期了就重新现查" + + +async def test_a_failed_probe_is_not_retried_for_a_while(monkeypatch): + """现查没成功（大表全表 MIN / MAX 超时）也记几分钟：期间同一张表直接退回剖析，不再每次白等探查的时限。""" + from types import SimpleNamespace + + seen: list[str] = [] + + async def probe(source, text): + seen.append(text) + return None + + async def facts(table): + return {"at": "2026-10-01T03:00:00+00:00", "date_ranges": [{"column": "event_date", "min": "20260930", + "max": "20260930"}]} + + monkeypatch.setattr(coverage, "_probe", probe) + monkeypatch.setattr(coverage, "_LIVE", {}) + source = SimpleNamespace(name=f"plant_{uuid.uuid4().hex[:6]}", kind="sqlite", schema_cache=None, id="s-failed") + payload = {"columns": ["n"], "rows": [[0]], "row_count": 1, "column_types": {"n": "number"}} + sql = "SELECT COUNT(*) AS n FROM gate_events WHERE event_date >= '20261012'" + for _ in range(3): + [gap] = await coverage.result_gaps(source, sql, payload, facts=facts) + assert gap["basis"] == "profile" + assert sum("MIN(" in t for t in seen) == 1, "失败的现查也记着，期间不再白等" + monkeypatch.setattr(coverage, "LIVE_TTL_S", 0.0) + coverage._LIVE.clear() + await coverage.result_gaps(source, sql, payload, facts=facts) + assert sum("MIN(" in t for t in seen) == 2, "过期了就重新现查" diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 070cc150..22c3c606 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -350,6 +350,13 @@ export const EVIDENCE_TEXT = { unmatchedPlain: '无法追溯的数字：', uncited: '该数字未使用引用标记，系统无法核对其来源', missingValue: '缺少输入：口径卡本次未计算出该指标的值', + /** 指标没有值、但不是缺输入：输入都齐，计算式自己算不出来（比如分母为 0） */ + undefinedValue: '算不出来：这组输入下计算式没有值（比如分母为 0），不是缺少输入', + /** 指标的输入取自「查询区间里这张表没有数据」的列：这个数不代表真的是 0，出具按缺口降档 */ + noData: '区间内无数据', + noDataFallback: '所依据的查询区间里这张表没有数据，这个数不代表真的是 0', + /** 查询步骤：这次查询里查询区间内表里没有数据的那几列 */ + noDataTitle: '查询区间内表里没有数据', /** 指标拿截断的查询结果整组算出来（计数、求和……）：值只算到了取回的那部分，出具按缺口降档 */ incomplete: '结果不完整', incompleteFallback: '基于被截断的查询结果计算，结果不完整', diff --git a/frontend/src/run/EvidencePanel.tsx b/frontend/src/run/EvidencePanel.tsx index bb33cc52..d9ebaa87 100644 --- a/frontend/src/run/EvidencePanel.tsx +++ b/frontend/src/run/EvidencePanel.tsx @@ -1361,6 +1361,8 @@ function MetricPart({ step, entry, inputs, runId, runClass, seal, pending, faile // 口径卡里 rendered 为「—」有两种：value 为 null 是缺输入，不为 null 是有值但按格式显示不出来 const dash = rendered === NONE || entry?.rendered === NONE const missing = entry?.status === 'missing_input' || (value == null && (dash || !resolved)) + // 没有值的两种：缺输入，和输入都齐、计算式自己算不出来（分母为 0，口径卡记了 undefined） + const undefinedValue = missing && (step?.undefined === true || entry?.undefined === true) const unshowable = !missing && dash && value != null // 拿截断的查询结果整组算出来的：值是真的算出来的，但只算到了取回的那部分，出具已按缺口降档 const incomplete = step?.incomplete === true || entry?.incomplete === true @@ -1368,6 +1370,9 @@ function MetricPart({ step, entry, inputs, runId, runClass, seal, pending, faile // 所依据的查询对照数据目录查出了错误级的问题：值照算，结果不可靠，出具同样按缺口降档。问题和改法在查询步骤里 const sqlFailed = step?.sql_check_failed === true || entry?.sql_check_failed === true const sqlReason = step?.sql_check_reason || entry?.sql_check_reason || EVIDENCE_TEXT.sqlCheckFallback + // 输入取自「查询区间里这张表没有数据」的列：值照算，那个 0 不代表真的是 0，出具同样按缺口降档 + const noData = step?.no_data === true || entry?.no_data === true + const noDataReason = step?.no_data_reason || entry?.no_data_reason || EVIDENCE_TEXT.noDataFallback // 期 4：双盲复算复核过这个指标才有 const dualCheck = dualCheckOf(step) // 输入追没追到原始数据：和「有出处」（确定性复算一致）分开说。卡上没有输入、老服务端没给时不说 @@ -1394,6 +1399,12 @@ function MetricPart({ step, entry, inputs, runId, runClass, seal, pending, faile {EVIDENCE_TEXT.sqlCheckFailed} )} + {noData && ( + + {EVIDENCE_TEXT.noData} + + )} {!!untraced?.length && ( @@ -1416,13 +1427,21 @@ function MetricPart({ step, entry, inputs, runId, runClass, seal, pending, faile )} {dualCheck && } - {missing &&

{EVIDENCE_TEXT.missingValue}

} + {missing && ( +

+ {undefinedValue ? EVIDENCE_TEXT.undefinedValue : EVIDENCE_TEXT.missingValue} +

+ )} {incomplete && (

{incompleteReason}

)} {sqlFailed && (

{sqlReason}

)} + {noData && ( +

{noDataReason}

+ )} {unshowable && (

{EVIDENCE_TEXT.unshowable(valueText(value))} @@ -1657,6 +1676,10 @@ function UpstreamPart({ id, step, seal, linkOf }: { {EVIDENCE_TEXT.sqlCheckFailed} )} + {step.no_data === true && ( + {EVIDENCE_TEXT.noData} + )} {host.onNode && } @@ -1854,6 +1877,20 @@ function QueryPart({ id, step, seal, masked, pick }: { )} + {Array.isArray(step.no_data) && step.no_data.length > 0 && ( + // 查询区间里这张表没有数据的那几列（查询时现查）：这些 0 不代表真的是 0。没有问题时接口不给这个键 +

+
{EVIDENCE_TEXT.noDataTitle}
+
    + {step.no_data.map((g, k) => ( +
  • + + {g.message} +
    • + ))} +
+
+ )} {!!step.merge?.warnings.length && (
{MERGE_TEXT.executionWarnings}
diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 98a04f08..85aa7987 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -2346,6 +2346,21 @@ export interface EvidenceMerge { } /** 指标的一个输入追不到原始数据：出处追到哪个节点为止。认不出是哪个节点时 node_id、label 为 null */ +/** 查询区间里这张表没有数据的那几列（服务端 data/coverage.result_gaps） */ +export interface EvidenceNoData { + columns: string[] + table?: string + column?: string + /** 查的区间（按天，两头都含），没有限制的那一头为 null */ + window?: (string | null)[] + /** 表里这一列的范围；表里一个值都没有时为 null */ + data_range?: (string | null)[] | null + /** live 现查；profile / observed 现查不成，按剖析、取值观察 */ + basis?: string + basis_at?: string + message: string +} + export interface EvidenceUntraced { path: string node_id: string | null @@ -2399,6 +2414,15 @@ export interface EvidenceStep extends Omit { sql_check_failed?: boolean /** metric：没通过的原因（人话，「所依据的查询未通过 SQL 检查（…），结果不可靠」） */ sql_check_reason?: string + /** metric：没有值，不是缺输入，是输入都齐、计算式自己算不出来（分母为 0）。状态仍是 missing_input */ + undefined?: boolean + /** + * metric：输入取自「查询区间里这张表没有数据」的列，值照算，出具按缺口降档（true）； + * query：这次查询里这样的几列（服务端 data/coverage.py，查询时记进快照）。没有问题时没有这个键 + */ + no_data?: boolean | EvidenceNoData[] + /** metric：没有数据的原因（人话，「所依据的查询区间里这张表没有数据：…」） */ + no_data_reason?: string /** metric（期 4）：这个指标的双盲复算状态。没有双盲复算复核过这份指标集时没有这个键 */ dual_check?: EvidenceDualCheck /** diff --git a/scripts/dialect_it.py b/scripts/dialect_it.py index f5621d42..362df5a6 100644 --- a/scripts/dialect_it.py +++ b/scripts/dialect_it.py @@ -149,6 +149,23 @@ def _ddl(t: Target) -> list[str]: return out +def _snapshot(t: Target) -> list[str]: + """覆盖检查（data/coverage.py）的夹具：只存当天的滚动表（每天整表重灌的那种）。日期按库自己的当前日期写，避开容器 + 和本机时区不一致；daily_snapshot 的日期按文字 'YYYYMMDD' 存（部署上撞过的写法，只在 MySQL / MariaDB 上判）， + daily_snapshot_d 按 DATE 存。""" + vc = "VARCHAR2" if t.oracle else "VARCHAR" + today_text = ("TO_CHAR(SYSDATE, 'YYYYMMDD')" if t.oracle + else "REPLACE(CAST(CURRENT_DATE AS CHAR), '-', '')" if t.mysql + else "REPLACE(CAST(CURRENT_DATE AS VARCHAR), '-', '')") + today = "TRUNC(SYSDATE)" if t.oracle else "CURRENT_DATE" + out = [f"CREATE TABLE daily_snapshot (event_day {vc}(8), visitor {vc}(16))", + f"CREATE TABLE daily_snapshot_d (event_day DATE, visitor {vc}(16))"] + for who in ("v1", "v2", "v3"): + out.append(f"INSERT INTO daily_snapshot (event_day, visitor) VALUES ({today_text}, '{who}')") + out.append(f"INSERT INTO daily_snapshot_d (event_day, visitor) VALUES ({today}, '{who}')") + return out + + def _paid(t: Target, o: dict[str, Any]) -> str: if o["paid_at"] is not None: stamp = o["paid_at"].strftime("%Y-%m-%d %H:%M:%S") @@ -212,7 +229,7 @@ async def _setup(t: Target) -> None: if t.mysql: # 关掉严格模式才写得进零日期（老库里常见的情形） await conn.execute(text("SET SESSION sql_mode = ''")) - for sql in _ddl(t) + _rows(t) + _events(1, N_EVENTS_OLD): + for sql in _ddl(t) + _rows(t) + _events(1, N_EVENTS_OLD) + _snapshot(t): await conn.exec_driver_sql(sql) async with engine.begin() as conn: for sql in _analyze(t, "events") + _analyze(t, "orders") + _analyze(t, "stores"): @@ -421,10 +438,41 @@ async def run_target(t: Target) -> None: res = await data_engine.run_query(source, d.max_date_sql(tbl, paid_col)) check(bool(res.rows) and str(res.rows[0][0])[:19].replace("T", " ") == str(EXPECT["paid_at"][1]), f"max_date_sql 直接跑：{res.rows}") + await _coverage(t, source) finally: await data_engine.engines.invalidate(source.id) +async def _coverage(t: Target, source: Any) -> None: + """覆盖检查（data/coverage.py）在真库上：按运行当天现算的上周区间，表里只有当天的数据 → 判成没有数据；区间包含当天、 + 只是别的条件筛空的 → 不报。界在库里算（各方言自己的写法），表的范围现查 MIN / MAX。""" + from app.data import coverage + + last_week = {"mysql": ("event_day >= DATE_SUB(CURRENT_DATE, INTERVAL (WEEKDAY(CURRENT_DATE) + 7) DAY) " + "AND event_day < DATE_SUB(CURRENT_DATE, INTERVAL WEEKDAY(CURRENT_DATE) DAY)"), + "pg": ("event_day >= date_trunc('week', CURRENT_DATE) - INTERVAL '7 days' " + "AND event_day < date_trunc('week', CURRENT_DATE)"), + "oracle": "event_day >= TRUNC(SYSDATE, 'IW') - 7 AND event_day < TRUNC(SYSDATE, 'IW')"} + recent = {"mysql": "event_day >= DATE_SUB(CURRENT_DATE, INTERVAL 1 DAY)", + "pg": "event_day >= CURRENT_DATE - INTERVAL '1 day'", + "oracle": "event_day >= TRUNC(SYSDATE) - 1"} + flavor = "mysql" if t.mysql else "pg" if t.pg else "oracle" + today_sql = "SELECT TRUNC(SYSDATE) FROM DUAL" if t.oracle else "SELECT CURRENT_DATE" + today = str((await data_engine.run_query(source, today_sql)).rows[0][0])[:10] + for table in ["daily_snapshot_d"] + (["daily_snapshot"] if t.mysql else []): + sql = f"SELECT COUNT(DISTINCT visitor) AS n FROM {table} WHERE {last_week[flavor]}" + gaps = await coverage.result_gaps(source, sql, (await data_engine.run_query(source, sql)).to_payload()) + check(len(gaps) == 1 and gaps[0]["data_range"] == [today, today] + and [c.lower() for c in gaps[0]["columns"]] == ["n"], + f"覆盖检查：{table} 只有当天的数据，上周的区间判成没有数据：{gaps}") + pre = await coverage.sql_gaps(source, sql) + check(len(pre) == 1, f"助手自查：{table} 上周的区间判成没有数据：{pre}") + fine = f"SELECT COUNT(DISTINCT visitor) AS n FROM {table} WHERE visitor = 'nobody' AND {recent[flavor]}" + quiet = await coverage.result_gaps(source, fine, (await data_engine.run_query(source, fine)).to_payload()) + check(quiet == [], f"覆盖检查：{table} 区间里有数据、只是别的条件筛空的不报：{quiet}") + check(await coverage.sql_gaps(source, fine) == [], f"助手自查：{table} 区间包含当天的不报") + + async def main() -> int: only = set((os.environ.get("DIALECT_IT_ONLY") or "pg,mysql,mariadb,oracle").split(",")) plan = [("pg", "postgresql", "DIALECT_IT_PG_PORT"), ("mysql", "mysql", "DIALECT_IT_MYSQL_PORT"),