口径取数 SQL 增加书写 lint:取整、补 0 在画布提示并在受管发布时拦截

This commit is contained in:
ThearchyHelios
2026-11-03 18:36:26 +08:00
parent 80a479d723
commit ce2f68a373
4 changed files with 293 additions and 2 deletions
+116
View File
@@ -0,0 +1,116 @@
"""口径取数 SQL 的书写 lint:取整、补 0 应当交给口径卡,不写进取数 SQL。
口径账本(ledger/match)证明「问数据的查询与口径卡等价」时只认规范化后完全相同的结果。口径卡的取数 SQL 里写了
ROUND(...),未取整的查询无论数值对不对,规范化后都多一层取整;写了 COALESCE(..., 0),没有数据时算出 0,两边
永远对不上。所以这两处要在画布校验和发布门禁里提示/拦截——不能在比对时放松,放松就认不出真正不同的口径了。
只做 SQL 级检查,不读 GraphSpec:「哪些节点是口径卡的上游取数节点」由 engine 侧找(engine/schema.py)。
"""
from __future__ import annotations
from typing import Any
ROUND_IN_SOURCE = "caliber.round_in_source"
ZERO_DEFAULT = "caliber.zero_default"
ROUND_MESSAGE = (
"取数 SQL 里写了 ROUND(...):取整交给口径卡的小数位和展示;取数 SQL 里写 ROUND 会让未取整的查询"
"永远认不成等价(口径账本只认规范化后完全相同的结果)"
)
ZERO_MESSAGE = (
"取数 SQL 里用 COALESCE / IFNULL / NVL 给缺省值补了 0:缺省值交给口径卡;取数里补 0 会把「没有数据」"
"算成 0,且永远认不成等价(口径账本只认规范化后完全相同的结果)"
)
#: excerpt 的截断长度:够看清出问题的那一段,又不至于把整条查询塞进提示
EXCERPT_CHARS = 200
_ZERO_FUNCS = frozenset({"COALESCE", "IFNULL", "NVL"})
def issues_for_sql(sql: str) -> list[dict[str, Any]]:
"""检查一条取数 SQL 的书写问题:[{code, message, excerpt}]。
解析不了、不是字符串、空串都返回空列表:lint 是附加提示,不能挡住主流程。同时有 ROUND 和补 0 时按它们在
SQL 里出现的先后返回;调用方每个取数节点只取第一条。
"""
if not isinstance(sql, str) or not sql.strip():
return []
try:
import sqlglot
from sqlglot import exp
except ImportError:
return []
try:
tree = sqlglot.parse_one(sql)
except Exception: # noqa: BLE001 - 语法错、方言特性、sqlglot 不支持的写法:一律不查
return []
if tree is None:
return []
rounds = list(tree.find_all(exp.Round))
coalesces = list(tree.find_all(exp.Coalesce))
# 兜底按函数名认:某些方言/写法下 sqlglot 不把函数落到专门的节点上(Anonymous),节点名才是唯一的依据
for func in tree.find_all(exp.Func):
name = _function_name(func, exp)
if name == "ROUND" and not isinstance(func, exp.Round):
rounds.append(func)
elif name in _ZERO_FUNCS and not isinstance(func, exp.Coalesce):
coalesces.append(func)
found: list[tuple[int, dict[str, Any]]] = []
found.extend((_position(func), _issue(ROUND_IN_SOURCE, ROUND_MESSAGE, func)) for func in rounds)
found.extend((_position(func), _issue(ZERO_DEFAULT, ZERO_MESSAGE, func))
for func in coalesces if _ends_with_zero(func, exp))
found.sort(key=lambda item: item[0])
return [issue for _, issue in found]
def _function_name(func: Any, exp: Any) -> str:
"""函数名的大写写法。Anonymous 的 sql_name() 只给出 ANONYMOUS,要看它自己记的名字。"""
name = func.sql_name()
if isinstance(func, exp.Anonymous) and isinstance(func.this, str):
name = func.this
return str(name or "").upper()
def _arguments(func: Any, exp: Any) -> list[Any]:
"""函数的参数列表。Coalesce 把第一个参数放在 this 上,其余在 expressions。"""
if isinstance(func, exp.Coalesce):
return [func.this, *func.expressions]
return list(func.expressions)
def _ends_with_zero(func: Any, exp: Any) -> bool:
"""最后一个参数是数值 0:COALESCE(x, 0) / IFNULL(x, 0) / NVL(x, 0)。
只认常量 0:COALESCE(x, y) 是正常的缺省列,不该报。
"""
args = _arguments(func, exp)
if len(args) < 2 or args[-1] is None:
return False
last = args[-1]
while isinstance(last, (exp.Paren, exp.Neg)):
last = last.this
if not isinstance(last, exp.Literal) or not last.is_number:
return False
try:
return float(last.name) == 0
except (TypeError, ValueError):
return False
def _position(func: Any) -> int:
"""节点在原文里的起始位置,用来按出现先后排序;拿不到位置的排在最后。"""
start = (getattr(func, "meta", None) or {}).get("start")
return int(start) if isinstance(start, int) else 1 << 30
def _issue(code: str, message: str, func: Any) -> dict[str, Any]:
return {"code": code, "message": message, "excerpt": _excerpt(func.sql())}
def _excerpt(text: str) -> str:
text = " ".join(str(text).split())
return text if len(text) <= EXCERPT_CHARS else text[:EXCERPT_CHARS - 1] + "…"
+19 -2
View File
@@ -11,6 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.engine.issuance import metrics_only
from app.engine.labels import contract_label, field_label, judge_label, option_label, q
from app.engine.schema import (
CALIBER_SQL_CODES,
CLAIMS_JUDGE,
GraphNode,
GraphSpec,
@@ -19,6 +20,7 @@ from app.engine.schema import (
_ancestors,
_caliber_inputs,
analysis_claims_problem,
caliber_sql_issues,
claims_problem,
contract_needs_metrics,
judge_problems,
@@ -132,6 +134,7 @@ def lint_for_publish(spec: GraphSpec, *, level: str, checkers: Mapping[str, Any]
_lint_evidence(spec, flag, strict=strict)
_lint_reject_exits(spec, flag)
_lint_sql(spec, flag, checkers)
_lint_caliber_sql(spec, flag)
_load_extra_lints()
for check in EXTRA_LINTS:
check(spec, flag, level=level, strict=strict)
@@ -215,9 +218,22 @@ def _lint_sql(spec: GraphSpec, flag: Callable[..., None], checkers: Mapping[str,
sql_level=check.level)
def _lint_caliber_sql(spec: GraphSpec, flag: Callable[..., None]) -> None:
"""口径卡上游取数节点的 SQL 书写(data/caliber_lint.py):ROUND、COALESCE(…, 0)。
取整、缺省值应当交给口径卡:取数 SQL 里写了它们,口径账本(ledger/match)永远认不出未取整的查询与口径等价。
已发布档只警告,受管档挡住发布——和别的受管要求同一个口径。定义来自上游的卡(caliber_from)不在本图,跳过。
"""
for feeder, problem in caliber_sql_issues(spec):
flag(f"取数节点「{feeder.title}」:{problem['message']}({problem['excerpt']})",
code=problem["code"], node_id=feeder.id, hard=True, field="args.sql")
#: validate 里和门禁 G4、G5 说同一件事的两条提示(schema.evidence_issues),和受管级分析卡输入那条(governed.analysis_input)
#: 说同一件事的画布提醒(schema.analysis_issues 对调用别的工具的「调用工具」节点的那条)
_REPEATS = frozenset({"evidence.caliber_compute_input", "evidence.cite_fields_off", "analysis.input_invalid"})
#: 说同一件事的画布提醒(schema.analysis_issues 对调用别的工具的「调用工具」节点的那条),以及口径取数 SQL 的两条
#: lint(门禁和画布说的是同一条 SQL、同一个字段)
_REPEATS = frozenset({"evidence.caliber_compute_input", "evidence.cite_fields_off", "analysis.input_invalid",
*CALIBER_SQL_CODES})
def _not_repeated(issues: list[Any], gate: list[Any], spec: GraphSpec) -> list[Any]:
@@ -234,6 +250,7 @@ def _not_repeated(issues: list[Any], gate: list[Any], spec: GraphSpec) -> list[A
compute = {i.node_id for i in gate if i.code == "governed.caliber_compute_input"}
covered = {("evidence.cite_fields_off", nid, "output_schema") for nid in cited}
covered |= {("analysis.input_invalid", i.node_id, i.field) for i in gate if i.code == "governed.analysis_input"}
covered |= {(i.code, i.node_id, i.field) for i in gate if i.code in CALIBER_SQL_CODES}
for card in spec.nodes:
if card.type == NodeType.METRICS and compute:
covered |= {("evidence.caliber_compute_input", card.id, f"metrics[{i}].expression")
+41
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import math
import re
from collections import defaultdict
from collections.abc import Mapping
from enum import StrEnum
from typing import Any, Iterable, Literal
@@ -1645,6 +1646,9 @@ def text_parse_issues(spec: GraphSpec) -> list[ValidationIssue]:
EVIDENCE_ROLES = ("source", "compute")
_REF_HEAD = re.compile(r"^(vars|nodes)\.([\w-]+)")
#: 口径卡上游取数 SQL 的书写问题(data/caliber_lint.py):取整、补 0 应交给口径卡
CALIBER_SQL_CODES = frozenset({"caliber.round_in_source", "caliber.zero_default"})
def _caliber_inputs(card: GraphNode, spec: GraphSpec) -> list[tuple[int, GraphNode]]:
"""口径卡每个指标表达式里读到的节点:(指标下标, 产出它的节点)。"""
@@ -1670,6 +1674,40 @@ def _caliber_inputs(card: GraphNode, spec: GraphSpec) -> list[tuple[int, GraphNo
return out
def caliber_sql_issues(spec: GraphSpec) -> list[tuple[GraphNode, dict[str, Any]]]:
"""口径卡上游取数节点里写坏了的取数 SQL:(取数节点, 问题),每个节点最多一条。
取数 SQL 里的 ROUND、COALESCE(…, 0) 让未取整的查询永远认不成等价(ledger/match 只认规范化后完全相同),
取整、缺省值应当交给口径卡。定义来自上游的卡(caliber_from)不在本图,跳过;调用工具节点照 ledger/match
的认法:工具名以 QUERY_PREFIX 开头、SQL 里没有「{{」运行时模板。
"""
from app.data.caliber_lint import issues_for_sql
from app.tools.datasource import QUERY_PREFIX
out: list[tuple[GraphNode, dict[str, Any]]] = []
seen: set[str] = set()
for card in spec.nodes:
if card.type != NodeType.METRICS or card.config.get("caliber_from") not in (None, "", {}):
continue
for _, feeder in _caliber_inputs(card, spec):
if feeder.id in seen:
continue
seen.add(feeder.id)
if feeder.type != NodeType.TOOL:
continue
cfg = feeder.config
args = cfg.get("args")
sql = args.get("sql") if isinstance(args, Mapping) else None
if not str(cfg.get("tool") or "").startswith(QUERY_PREFIX):
continue
if not isinstance(sql, str) or not sql.strip() or "{{" in sql:
continue
found = issues_for_sql(sql)
if found:
out.append((feeder, found[0]))
return out
def _is_zero(node: Any) -> bool:
"""常量 0(0、0.0、-0、+0);布尔不算。"""
import ast
@@ -1745,6 +1783,9 @@ def evidence_issues(spec: GraphSpec) -> list[ValidationIssue]:
"无法追溯出处。负责取数的代码节点请将「证据角色」设为「取数」;业务计算请移到口径卡的表达式中",
node, f"metrics[{i}].expression",
code="evidence.caliber_compute_input")
for feeder, problem in caliber_sql_issues(spec):
add(f"取数节点「{feeder.title}」:{problem['message']}({problem['excerpt']})",
feeder, "args.sql", code=problem["code"])
return out
+117
View File
@@ -0,0 +1,117 @@
"""口径取数 SQL 的书写 lint:取整、补 0 交给口径卡,画布提示、发布门禁拦截。
- 口径卡上游取数 SQL 里的 ROUND(...)、COALESCE / IFNULL / NVL(..., 0):evidence_issues 在取数节点上给 warning
- 受管发布门禁报 error,已发布档只 warning:和别的受管要求同一个口径
- 解析不了的 SQL 不抛异常;定义来自上游的卡(caliber_from)不在本图,不查
"""
from __future__ import annotations
import pytest
from app.data.caliber_lint import issues_for_sql
from app.engine.governance import lint_for_publish, publish_issues
from app.engine.schema import GraphSpec, evidence_issues
ROUND = "caliber.round_in_source"
ZERO = "caliber.zero_default"
CELL = "cell(nodes.fetch, 0, 'gmv')"
def node(nid, ntype, label=None, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
"data": {"label": label or nid, "config": config}}
def graph(sql: str, *, metrics=None, **card) -> dict:
"""input → 取数 → 口径卡。默认两个指标都读同一个取数节点,用来验证同一节点只报一次。"""
default = [{"id": "gmv", "name": "销售额", "unit": "元", "expression": CELL},
{"id": "orders", "name": "订单数", "unit": "单", "expression": CELL}]
nodes = [
node("start", "input", "周期"),
node("fetch", "tool", "取数", tool="db_query__shop", args={"sql": sql}),
node("card", "metrics", "周报口径卡", metrics=metrics or default, **card),
]
return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]}
for a, b in zip(nodes, nodes[1:])]}
def gate(graph_dict: dict, code: str, level: str = "governed") -> list:
return [i for i in lint_for_publish(GraphSpec.model_validate(graph_dict), level=level).issues
if i.code == code]
# --------------------------------------------------------------------------
# SQL 级检查
# --------------------------------------------------------------------------
def test_round_in_source_is_detected():
found = issues_for_sql("SELECT ROUND(SUM(amount), 2) AS gmv FROM orders")
assert [i["code"] for i in found] == [ROUND]
assert found[0]["excerpt"] == "ROUND(SUM(amount), 2)"
assert "永远认不成等价" in found[0]["message"]
def test_zero_default_needs_a_literal_zero_last_argument():
found = issues_for_sql("SELECT COALESCE(SUM(amount), 0) AS gmv FROM orders")
assert [i["code"] for i in found] == [ZERO]
assert "没有数据" in found[0]["message"]
assert [i["code"] for i in issues_for_sql("SELECT COALESCE(SUM(amount), -0) AS gmv FROM orders")] == [ZERO]
assert issues_for_sql("SELECT COALESCE(SUM(amount), SUM(refund)) AS gmv FROM orders") == []
assert issues_for_sql("SELECT COALESCE(SUM(amount), 1) AS gmv FROM orders") == []
@pytest.mark.parametrize("sql", ["SELECT IFNULL(SUM(amount), 0) AS gmv FROM orders",
"SELECT NVL(SUM(amount), 0) AS gmv FROM orders"])
def test_ifnull_and_nvl_count_as_zero_default(sql):
assert [i["code"] for i in issues_for_sql(sql)] == [ZERO]
def test_unparsable_sql_is_ignored():
assert issues_for_sql("SELECT (((") == []
assert issues_for_sql("") == []
assert issues_for_sql(None) == [] # type: ignore[arg-type]
# --------------------------------------------------------------------------
# 画布校验:落在取数节点上,一个节点只报一次
# --------------------------------------------------------------------------
def test_evidence_warns_on_the_fetch_node_once():
spec = GraphSpec.model_validate(graph("SELECT ROUND(SUM(amount), 2) AS gmv FROM orders"))
found = [i for i in evidence_issues(spec) if i.code == ROUND]
assert len(found) == 1
assert found[0].level == "warning"
assert found[0].node_id == "fetch"
assert found[0].field == "args.sql"
def test_evidence_skips_cards_from_upstream():
ref = {"workflow_id": "lib", "workflow_version": 1, "node_id": "card"}
spec = GraphSpec.model_validate(
graph("SELECT COALESCE(SUM(amount), 0) AS gmv FROM orders", caliber_from=ref))
assert [i for i in evidence_issues(spec) if i.code in (ROUND, ZERO)] == []
# --------------------------------------------------------------------------
# 发布门禁:已发布只警告,受管挡住
# --------------------------------------------------------------------------
@pytest.mark.parametrize("code,sql", [
(ROUND, "SELECT ROUND(SUM(amount), 2) AS gmv FROM orders"),
(ZERO, "SELECT COALESCE(SUM(amount), 0) AS gmv FROM orders"),
])
def test_governed_blocks_and_published_warns(code, sql):
graph_dict = graph(sql)
assert [i.level for i in gate(graph_dict, code, "governed")] == ["error"]
assert [i.level for i in gate(graph_dict, code, "published")] == ["warning"]
def test_publish_issues_lists_the_problem_once():
"""validate 和门禁说的是同一处:发布时不能一正一反各来一条。"""
spec = GraphSpec.model_validate(graph("SELECT ROUND(SUM(amount), 2) AS gmv FROM orders"))
found = [i for i in publish_issues(spec, level="published") if i.code == ROUND]
assert len(found) == 1
assert found[0].level == "warning"