mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-19 08:22:00 +00:00
口径取数 SQL 增加书写 lint:取整、补 0 在画布提示并在受管发布时拦截
This commit is contained in:
@@ -0,0 +1,116 @@
|
||||
"""口径取数 SQL 的书写 lint:取整、补 0 应当交给口径卡,不写进取数 SQL。
|
||||
|
||||
口径账本(ledger/match)证明「问数据的查询与口径卡等价」时只认规范化后完全相同的结果。口径卡的取数 SQL 里写了
|
||||
ROUND(...),未取整的查询无论数值对不对,规范化后都多一层取整;写了 COALESCE(..., 0),没有数据时算出 0,两边
|
||||
永远对不上。所以这两处要在画布校验和发布门禁里提示/拦截——不能在比对时放松,放松就认不出真正不同的口径了。
|
||||
|
||||
只做 SQL 级检查,不读 GraphSpec:「哪些节点是口径卡的上游取数节点」由 engine 侧找(engine/schema.py)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
ROUND_IN_SOURCE = "caliber.round_in_source"
|
||||
ZERO_DEFAULT = "caliber.zero_default"
|
||||
|
||||
ROUND_MESSAGE = (
|
||||
"取数 SQL 里写了 ROUND(...):取整交给口径卡的小数位和展示;取数 SQL 里写 ROUND 会让未取整的查询"
|
||||
"永远认不成等价(口径账本只认规范化后完全相同的结果)"
|
||||
)
|
||||
ZERO_MESSAGE = (
|
||||
"取数 SQL 里用 COALESCE / IFNULL / NVL 给缺省值补了 0:缺省值交给口径卡;取数里补 0 会把「没有数据」"
|
||||
"算成 0,且永远认不成等价(口径账本只认规范化后完全相同的结果)"
|
||||
)
|
||||
|
||||
#: excerpt 的截断长度:够看清出问题的那一段,又不至于把整条查询塞进提示
|
||||
EXCERPT_CHARS = 200
|
||||
|
||||
_ZERO_FUNCS = frozenset({"COALESCE", "IFNULL", "NVL"})
|
||||
|
||||
|
||||
def issues_for_sql(sql: str) -> list[dict[str, Any]]:
|
||||
"""检查一条取数 SQL 的书写问题:[{code, message, excerpt}]。
|
||||
|
||||
解析不了、不是字符串、空串都返回空列表:lint 是附加提示,不能挡住主流程。同时有 ROUND 和补 0 时按它们在
|
||||
SQL 里出现的先后返回;调用方每个取数节点只取第一条。
|
||||
"""
|
||||
if not isinstance(sql, str) or not sql.strip():
|
||||
return []
|
||||
try:
|
||||
import sqlglot
|
||||
from sqlglot import exp
|
||||
except ImportError:
|
||||
return []
|
||||
try:
|
||||
tree = sqlglot.parse_one(sql)
|
||||
except Exception: # noqa: BLE001 - 语法错、方言特性、sqlglot 不支持的写法:一律不查
|
||||
return []
|
||||
if tree is None:
|
||||
return []
|
||||
|
||||
rounds = list(tree.find_all(exp.Round))
|
||||
coalesces = list(tree.find_all(exp.Coalesce))
|
||||
# 兜底按函数名认:某些方言/写法下 sqlglot 不把函数落到专门的节点上(Anonymous),节点名才是唯一的依据
|
||||
for func in tree.find_all(exp.Func):
|
||||
name = _function_name(func, exp)
|
||||
if name == "ROUND" and not isinstance(func, exp.Round):
|
||||
rounds.append(func)
|
||||
elif name in _ZERO_FUNCS and not isinstance(func, exp.Coalesce):
|
||||
coalesces.append(func)
|
||||
|
||||
found: list[tuple[int, dict[str, Any]]] = []
|
||||
found.extend((_position(func), _issue(ROUND_IN_SOURCE, ROUND_MESSAGE, func)) for func in rounds)
|
||||
found.extend((_position(func), _issue(ZERO_DEFAULT, ZERO_MESSAGE, func))
|
||||
for func in coalesces if _ends_with_zero(func, exp))
|
||||
found.sort(key=lambda item: item[0])
|
||||
return [issue for _, issue in found]
|
||||
|
||||
|
||||
def _function_name(func: Any, exp: Any) -> str:
|
||||
"""函数名的大写写法。Anonymous 的 sql_name() 只给出 ANONYMOUS,要看它自己记的名字。"""
|
||||
name = func.sql_name()
|
||||
if isinstance(func, exp.Anonymous) and isinstance(func.this, str):
|
||||
name = func.this
|
||||
return str(name or "").upper()
|
||||
|
||||
|
||||
def _arguments(func: Any, exp: Any) -> list[Any]:
|
||||
"""函数的参数列表。Coalesce 把第一个参数放在 this 上,其余在 expressions。"""
|
||||
if isinstance(func, exp.Coalesce):
|
||||
return [func.this, *func.expressions]
|
||||
return list(func.expressions)
|
||||
|
||||
|
||||
def _ends_with_zero(func: Any, exp: Any) -> bool:
|
||||
"""最后一个参数是数值 0:COALESCE(x, 0) / IFNULL(x, 0) / NVL(x, 0)。
|
||||
|
||||
只认常量 0:COALESCE(x, y) 是正常的缺省列,不该报。
|
||||
"""
|
||||
args = _arguments(func, exp)
|
||||
if len(args) < 2 or args[-1] is None:
|
||||
return False
|
||||
last = args[-1]
|
||||
while isinstance(last, (exp.Paren, exp.Neg)):
|
||||
last = last.this
|
||||
if not isinstance(last, exp.Literal) or not last.is_number:
|
||||
return False
|
||||
try:
|
||||
return float(last.name) == 0
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _position(func: Any) -> int:
|
||||
"""节点在原文里的起始位置,用来按出现先后排序;拿不到位置的排在最后。"""
|
||||
start = (getattr(func, "meta", None) or {}).get("start")
|
||||
return int(start) if isinstance(start, int) else 1 << 30
|
||||
|
||||
|
||||
def _issue(code: str, message: str, func: Any) -> dict[str, Any]:
|
||||
return {"code": code, "message": message, "excerpt": _excerpt(func.sql())}
|
||||
|
||||
|
||||
def _excerpt(text: str) -> str:
|
||||
text = " ".join(str(text).split())
|
||||
return text if len(text) <= EXCERPT_CHARS else text[:EXCERPT_CHARS - 1] + "…"
|
||||
@@ -11,6 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.engine.issuance import metrics_only
|
||||
from app.engine.labels import contract_label, field_label, judge_label, option_label, q
|
||||
from app.engine.schema import (
|
||||
CALIBER_SQL_CODES,
|
||||
CLAIMS_JUDGE,
|
||||
GraphNode,
|
||||
GraphSpec,
|
||||
@@ -19,6 +20,7 @@ from app.engine.schema import (
|
||||
_ancestors,
|
||||
_caliber_inputs,
|
||||
analysis_claims_problem,
|
||||
caliber_sql_issues,
|
||||
claims_problem,
|
||||
contract_needs_metrics,
|
||||
judge_problems,
|
||||
@@ -132,6 +134,7 @@ def lint_for_publish(spec: GraphSpec, *, level: str, checkers: Mapping[str, Any]
|
||||
_lint_evidence(spec, flag, strict=strict)
|
||||
_lint_reject_exits(spec, flag)
|
||||
_lint_sql(spec, flag, checkers)
|
||||
_lint_caliber_sql(spec, flag)
|
||||
_load_extra_lints()
|
||||
for check in EXTRA_LINTS:
|
||||
check(spec, flag, level=level, strict=strict)
|
||||
@@ -215,9 +218,22 @@ def _lint_sql(spec: GraphSpec, flag: Callable[..., None], checkers: Mapping[str,
|
||||
sql_level=check.level)
|
||||
|
||||
|
||||
def _lint_caliber_sql(spec: GraphSpec, flag: Callable[..., None]) -> None:
|
||||
"""口径卡上游取数节点的 SQL 书写(data/caliber_lint.py):ROUND、COALESCE(…, 0)。
|
||||
|
||||
取整、缺省值应当交给口径卡:取数 SQL 里写了它们,口径账本(ledger/match)永远认不出未取整的查询与口径等价。
|
||||
已发布档只警告,受管档挡住发布——和别的受管要求同一个口径。定义来自上游的卡(caliber_from)不在本图,跳过。
|
||||
"""
|
||||
for feeder, problem in caliber_sql_issues(spec):
|
||||
flag(f"取数节点「{feeder.title}」:{problem['message']}({problem['excerpt']})",
|
||||
code=problem["code"], node_id=feeder.id, hard=True, field="args.sql")
|
||||
|
||||
|
||||
#: validate 里和门禁 G4、G5 说同一件事的两条提示(schema.evidence_issues),和受管级分析卡输入那条(governed.analysis_input)
|
||||
#: 说同一件事的画布提醒(schema.analysis_issues 对调用别的工具的「调用工具」节点的那条)
|
||||
_REPEATS = frozenset({"evidence.caliber_compute_input", "evidence.cite_fields_off", "analysis.input_invalid"})
|
||||
#: 说同一件事的画布提醒(schema.analysis_issues 对调用别的工具的「调用工具」节点的那条),以及口径取数 SQL 的两条
|
||||
#: lint(门禁和画布说的是同一条 SQL、同一个字段)
|
||||
_REPEATS = frozenset({"evidence.caliber_compute_input", "evidence.cite_fields_off", "analysis.input_invalid",
|
||||
*CALIBER_SQL_CODES})
|
||||
|
||||
|
||||
def _not_repeated(issues: list[Any], gate: list[Any], spec: GraphSpec) -> list[Any]:
|
||||
@@ -234,6 +250,7 @@ def _not_repeated(issues: list[Any], gate: list[Any], spec: GraphSpec) -> list[A
|
||||
compute = {i.node_id for i in gate if i.code == "governed.caliber_compute_input"}
|
||||
covered = {("evidence.cite_fields_off", nid, "output_schema") for nid in cited}
|
||||
covered |= {("analysis.input_invalid", i.node_id, i.field) for i in gate if i.code == "governed.analysis_input"}
|
||||
covered |= {(i.code, i.node_id, i.field) for i in gate if i.code in CALIBER_SQL_CODES}
|
||||
for card in spec.nodes:
|
||||
if card.type == NodeType.METRICS and compute:
|
||||
covered |= {("evidence.caliber_compute_input", card.id, f"metrics[{i}].expression")
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import math
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping
|
||||
from enum import StrEnum
|
||||
from typing import Any, Iterable, Literal
|
||||
|
||||
@@ -1645,6 +1646,9 @@ def text_parse_issues(spec: GraphSpec) -> list[ValidationIssue]:
|
||||
EVIDENCE_ROLES = ("source", "compute")
|
||||
_REF_HEAD = re.compile(r"^(vars|nodes)\.([\w-]+)")
|
||||
|
||||
#: 口径卡上游取数 SQL 的书写问题(data/caliber_lint.py):取整、补 0 应交给口径卡
|
||||
CALIBER_SQL_CODES = frozenset({"caliber.round_in_source", "caliber.zero_default"})
|
||||
|
||||
|
||||
def _caliber_inputs(card: GraphNode, spec: GraphSpec) -> list[tuple[int, GraphNode]]:
|
||||
"""口径卡每个指标表达式里读到的节点:(指标下标, 产出它的节点)。"""
|
||||
@@ -1670,6 +1674,40 @@ def _caliber_inputs(card: GraphNode, spec: GraphSpec) -> list[tuple[int, GraphNo
|
||||
return out
|
||||
|
||||
|
||||
def caliber_sql_issues(spec: GraphSpec) -> list[tuple[GraphNode, dict[str, Any]]]:
|
||||
"""口径卡上游取数节点里写坏了的取数 SQL:(取数节点, 问题),每个节点最多一条。
|
||||
|
||||
取数 SQL 里的 ROUND、COALESCE(…, 0) 让未取整的查询永远认不成等价(ledger/match 只认规范化后完全相同),
|
||||
取整、缺省值应当交给口径卡。定义来自上游的卡(caliber_from)不在本图,跳过;调用工具节点照 ledger/match
|
||||
的认法:工具名以 QUERY_PREFIX 开头、SQL 里没有「{{」运行时模板。
|
||||
"""
|
||||
from app.data.caliber_lint import issues_for_sql
|
||||
from app.tools.datasource import QUERY_PREFIX
|
||||
|
||||
out: list[tuple[GraphNode, dict[str, Any]]] = []
|
||||
seen: set[str] = set()
|
||||
for card in spec.nodes:
|
||||
if card.type != NodeType.METRICS or card.config.get("caliber_from") not in (None, "", {}):
|
||||
continue
|
||||
for _, feeder in _caliber_inputs(card, spec):
|
||||
if feeder.id in seen:
|
||||
continue
|
||||
seen.add(feeder.id)
|
||||
if feeder.type != NodeType.TOOL:
|
||||
continue
|
||||
cfg = feeder.config
|
||||
args = cfg.get("args")
|
||||
sql = args.get("sql") if isinstance(args, Mapping) else None
|
||||
if not str(cfg.get("tool") or "").startswith(QUERY_PREFIX):
|
||||
continue
|
||||
if not isinstance(sql, str) or not sql.strip() or "{{" in sql:
|
||||
continue
|
||||
found = issues_for_sql(sql)
|
||||
if found:
|
||||
out.append((feeder, found[0]))
|
||||
return out
|
||||
|
||||
|
||||
def _is_zero(node: Any) -> bool:
|
||||
"""常量 0(0、0.0、-0、+0);布尔不算。"""
|
||||
import ast
|
||||
@@ -1745,6 +1783,9 @@ def evidence_issues(spec: GraphSpec) -> list[ValidationIssue]:
|
||||
"无法追溯出处。负责取数的代码节点请将「证据角色」设为「取数」;业务计算请移到口径卡的表达式中",
|
||||
node, f"metrics[{i}].expression",
|
||||
code="evidence.caliber_compute_input")
|
||||
for feeder, problem in caliber_sql_issues(spec):
|
||||
add(f"取数节点「{feeder.title}」:{problem['message']}({problem['excerpt']})",
|
||||
feeder, "args.sql", code=problem["code"])
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
"""口径取数 SQL 的书写 lint:取整、补 0 交给口径卡,画布提示、发布门禁拦截。
|
||||
|
||||
- 口径卡上游取数 SQL 里的 ROUND(...)、COALESCE / IFNULL / NVL(..., 0):evidence_issues 在取数节点上给 warning
|
||||
- 受管发布门禁报 error,已发布档只 warning:和别的受管要求同一个口径
|
||||
- 解析不了的 SQL 不抛异常;定义来自上游的卡(caliber_from)不在本图,不查
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.data.caliber_lint import issues_for_sql
|
||||
from app.engine.governance import lint_for_publish, publish_issues
|
||||
from app.engine.schema import GraphSpec, evidence_issues
|
||||
|
||||
ROUND = "caliber.round_in_source"
|
||||
ZERO = "caliber.zero_default"
|
||||
CELL = "cell(nodes.fetch, 0, 'gmv')"
|
||||
|
||||
|
||||
def node(nid, ntype, label=None, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
|
||||
"data": {"label": label or nid, "config": config}}
|
||||
|
||||
|
||||
def graph(sql: str, *, metrics=None, **card) -> dict:
|
||||
"""input → 取数 → 口径卡。默认两个指标都读同一个取数节点,用来验证同一节点只报一次。"""
|
||||
default = [{"id": "gmv", "name": "销售额", "unit": "元", "expression": CELL},
|
||||
{"id": "orders", "name": "订单数", "unit": "单", "expression": CELL}]
|
||||
nodes = [
|
||||
node("start", "input", "周期"),
|
||||
node("fetch", "tool", "取数", tool="db_query__shop", args={"sql": sql}),
|
||||
node("card", "metrics", "周报口径卡", metrics=metrics or default, **card),
|
||||
]
|
||||
return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]}
|
||||
for a, b in zip(nodes, nodes[1:])]}
|
||||
|
||||
|
||||
def gate(graph_dict: dict, code: str, level: str = "governed") -> list:
|
||||
return [i for i in lint_for_publish(GraphSpec.model_validate(graph_dict), level=level).issues
|
||||
if i.code == code]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# SQL 级检查
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_round_in_source_is_detected():
|
||||
found = issues_for_sql("SELECT ROUND(SUM(amount), 2) AS gmv FROM orders")
|
||||
assert [i["code"] for i in found] == [ROUND]
|
||||
assert found[0]["excerpt"] == "ROUND(SUM(amount), 2)"
|
||||
assert "永远认不成等价" in found[0]["message"]
|
||||
|
||||
|
||||
def test_zero_default_needs_a_literal_zero_last_argument():
|
||||
found = issues_for_sql("SELECT COALESCE(SUM(amount), 0) AS gmv FROM orders")
|
||||
assert [i["code"] for i in found] == [ZERO]
|
||||
assert "没有数据" in found[0]["message"]
|
||||
assert [i["code"] for i in issues_for_sql("SELECT COALESCE(SUM(amount), -0) AS gmv FROM orders")] == [ZERO]
|
||||
assert issues_for_sql("SELECT COALESCE(SUM(amount), SUM(refund)) AS gmv FROM orders") == []
|
||||
assert issues_for_sql("SELECT COALESCE(SUM(amount), 1) AS gmv FROM orders") == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", ["SELECT IFNULL(SUM(amount), 0) AS gmv FROM orders",
|
||||
"SELECT NVL(SUM(amount), 0) AS gmv FROM orders"])
|
||||
def test_ifnull_and_nvl_count_as_zero_default(sql):
|
||||
assert [i["code"] for i in issues_for_sql(sql)] == [ZERO]
|
||||
|
||||
|
||||
def test_unparsable_sql_is_ignored():
|
||||
assert issues_for_sql("SELECT (((") == []
|
||||
assert issues_for_sql("") == []
|
||||
assert issues_for_sql(None) == [] # type: ignore[arg-type]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 画布校验:落在取数节点上,一个节点只报一次
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_evidence_warns_on_the_fetch_node_once():
|
||||
spec = GraphSpec.model_validate(graph("SELECT ROUND(SUM(amount), 2) AS gmv FROM orders"))
|
||||
found = [i for i in evidence_issues(spec) if i.code == ROUND]
|
||||
assert len(found) == 1
|
||||
assert found[0].level == "warning"
|
||||
assert found[0].node_id == "fetch"
|
||||
assert found[0].field == "args.sql"
|
||||
|
||||
|
||||
def test_evidence_skips_cards_from_upstream():
|
||||
ref = {"workflow_id": "lib", "workflow_version": 1, "node_id": "card"}
|
||||
spec = GraphSpec.model_validate(
|
||||
graph("SELECT COALESCE(SUM(amount), 0) AS gmv FROM orders", caliber_from=ref))
|
||||
assert [i for i in evidence_issues(spec) if i.code in (ROUND, ZERO)] == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 发布门禁:已发布只警告,受管挡住
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code,sql", [
|
||||
(ROUND, "SELECT ROUND(SUM(amount), 2) AS gmv FROM orders"),
|
||||
(ZERO, "SELECT COALESCE(SUM(amount), 0) AS gmv FROM orders"),
|
||||
])
|
||||
def test_governed_blocks_and_published_warns(code, sql):
|
||||
graph_dict = graph(sql)
|
||||
assert [i.level for i in gate(graph_dict, code, "governed")] == ["error"]
|
||||
assert [i.level for i in gate(graph_dict, code, "published")] == ["warning"]
|
||||
|
||||
|
||||
def test_publish_issues_lists_the_problem_once():
|
||||
"""validate 和门禁说的是同一处:发布时不能一正一反各来一条。"""
|
||||
spec = GraphSpec.model_validate(graph("SELECT ROUND(SUM(amount), 2) AS gmv FROM orders"))
|
||||
found = [i for i in publish_issues(spec, level="published") if i.code == ROUND]
|
||||
assert len(found) == 1
|
||||
assert found[0].level == "warning"
|
||||
Reference in New Issue
Block a user