diff --git a/backend/app/data/caliber_lint.py b/backend/app/data/caliber_lint.py new file mode 100644 index 00000000..2e3614ee --- /dev/null +++ b/backend/app/data/caliber_lint.py @@ -0,0 +1,116 @@ +"""口径取数 SQL 的书写 lint:取整、补 0 应当交给口径卡,不写进取数 SQL。 + +口径账本(ledger/match)证明「问数据的查询与口径卡等价」时只认规范化后完全相同的结果。口径卡的取数 SQL 里写了 +ROUND(...),未取整的查询无论数值对不对,规范化后都多一层取整;写了 COALESCE(..., 0),没有数据时算出 0,两边 +永远对不上。所以这两处要在画布校验和发布门禁里提示/拦截——不能在比对时放松,放松就认不出真正不同的口径了。 + +只做 SQL 级检查,不读 GraphSpec:「哪些节点是口径卡的上游取数节点」由 engine 侧找(engine/schema.py)。 +""" + +from __future__ import annotations + +from typing import Any + +ROUND_IN_SOURCE = "caliber.round_in_source" +ZERO_DEFAULT = "caliber.zero_default" + +ROUND_MESSAGE = ( + "取数 SQL 里写了 ROUND(...):取整交给口径卡的小数位和展示;取数 SQL 里写 ROUND 会让未取整的查询" + "永远认不成等价(口径账本只认规范化后完全相同的结果)" +) +ZERO_MESSAGE = ( + "取数 SQL 里用 COALESCE / IFNULL / NVL 给缺省值补了 0:缺省值交给口径卡;取数里补 0 会把「没有数据」" + "算成 0,且永远认不成等价(口径账本只认规范化后完全相同的结果)" +) + +#: excerpt 的截断长度:够看清出问题的那一段,又不至于把整条查询塞进提示 +EXCERPT_CHARS = 200 + +_ZERO_FUNCS = frozenset({"COALESCE", "IFNULL", "NVL"}) + + +def issues_for_sql(sql: str) -> list[dict[str, Any]]: + """检查一条取数 SQL 的书写问题:[{code, message, excerpt}]。 + + 解析不了、不是字符串、空串都返回空列表:lint 是附加提示,不能挡住主流程。同时有 ROUND 和补 0 时按它们在 + SQL 里出现的先后返回;调用方每个取数节点只取第一条。 + """ + if not isinstance(sql, str) or not sql.strip(): + return [] + try: + import sqlglot + from sqlglot import exp + except ImportError: + return [] + try: + tree = sqlglot.parse_one(sql) + except Exception: # noqa: BLE001 - 语法错、方言特性、sqlglot 不支持的写法:一律不查 + return [] + if tree is None: + return [] + + rounds = list(tree.find_all(exp.Round)) + coalesces = list(tree.find_all(exp.Coalesce)) + # 兜底按函数名认:某些方言/写法下 sqlglot 不把函数落到专门的节点上(Anonymous),节点名才是唯一的依据 + for func in tree.find_all(exp.Func): + name = _function_name(func, exp) + if name == "ROUND" and not isinstance(func, exp.Round): + rounds.append(func) + elif name in _ZERO_FUNCS and not isinstance(func, exp.Coalesce): + coalesces.append(func) + + found: list[tuple[int, dict[str, Any]]] = [] + found.extend((_position(func), _issue(ROUND_IN_SOURCE, ROUND_MESSAGE, func)) for func in rounds) + found.extend((_position(func), _issue(ZERO_DEFAULT, ZERO_MESSAGE, func)) + for func in coalesces if _ends_with_zero(func, exp)) + found.sort(key=lambda item: item[0]) + return [issue for _, issue in found] + + +def _function_name(func: Any, exp: Any) -> str: + """函数名的大写写法。Anonymous 的 sql_name() 只给出 ANONYMOUS,要看它自己记的名字。""" + name = func.sql_name() + if isinstance(func, exp.Anonymous) and isinstance(func.this, str): + name = func.this + return str(name or "").upper() + + +def _arguments(func: Any, exp: Any) -> list[Any]: + """函数的参数列表。Coalesce 把第一个参数放在 this 上,其余在 expressions。""" + if isinstance(func, exp.Coalesce): + return [func.this, *func.expressions] + return list(func.expressions) + + +def _ends_with_zero(func: Any, exp: Any) -> bool: + """最后一个参数是数值 0:COALESCE(x, 0) / IFNULL(x, 0) / NVL(x, 0)。 + + 只认常量 0:COALESCE(x, y) 是正常的缺省列,不该报。 + """ + args = _arguments(func, exp) + if len(args) < 2 or args[-1] is None: + return False + last = args[-1] + while isinstance(last, (exp.Paren, exp.Neg)): + last = last.this + if not isinstance(last, exp.Literal) or not last.is_number: + return False + try: + return float(last.name) == 0 + except (TypeError, ValueError): + return False + + +def _position(func: Any) -> int: + """节点在原文里的起始位置,用来按出现先后排序;拿不到位置的排在最后。""" + start = (getattr(func, "meta", None) or {}).get("start") + return int(start) if isinstance(start, int) else 1 << 30 + + +def _issue(code: str, message: str, func: Any) -> dict[str, Any]: + return {"code": code, "message": message, "excerpt": _excerpt(func.sql())} + + +def _excerpt(text: str) -> str: + text = " ".join(str(text).split()) + return text if len(text) <= EXCERPT_CHARS else text[:EXCERPT_CHARS - 1] + "…" diff --git a/backend/app/engine/governance.py b/backend/app/engine/governance.py index 9bad2d8f..1e80b728 100644 --- a/backend/app/engine/governance.py +++ b/backend/app/engine/governance.py @@ -11,6 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.engine.issuance import metrics_only from app.engine.labels import contract_label, field_label, judge_label, option_label, q from app.engine.schema import ( + CALIBER_SQL_CODES, CLAIMS_JUDGE, GraphNode, GraphSpec, @@ -19,6 +20,7 @@ from app.engine.schema import ( _ancestors, _caliber_inputs, analysis_claims_problem, + caliber_sql_issues, claims_problem, contract_needs_metrics, judge_problems, @@ -132,6 +134,7 @@ def lint_for_publish(spec: GraphSpec, *, level: str, checkers: Mapping[str, Any] _lint_evidence(spec, flag, strict=strict) _lint_reject_exits(spec, flag) _lint_sql(spec, flag, checkers) + _lint_caliber_sql(spec, flag) _load_extra_lints() for check in EXTRA_LINTS: check(spec, flag, level=level, strict=strict) @@ -215,9 +218,22 @@ def _lint_sql(spec: GraphSpec, flag: Callable[..., None], checkers: Mapping[str, sql_level=check.level) +def _lint_caliber_sql(spec: GraphSpec, flag: Callable[..., None]) -> None: + """口径卡上游取数节点的 SQL 书写(data/caliber_lint.py):ROUND、COALESCE(…, 0)。 + + 取整、缺省值应当交给口径卡:取数 SQL 里写了它们,口径账本(ledger/match)永远认不出未取整的查询与口径等价。 + 已发布档只警告,受管档挡住发布——和别的受管要求同一个口径。定义来自上游的卡(caliber_from)不在本图,跳过。 + """ + for feeder, problem in caliber_sql_issues(spec): + flag(f"取数节点「{feeder.title}」:{problem['message']}({problem['excerpt']})", + code=problem["code"], node_id=feeder.id, hard=True, field="args.sql") + + #: validate 里和门禁 G4、G5 说同一件事的两条提示(schema.evidence_issues),和受管级分析卡输入那条(governed.analysis_input) -#: 说同一件事的画布提醒(schema.analysis_issues 对调用别的工具的「调用工具」节点的那条) -_REPEATS = frozenset({"evidence.caliber_compute_input", "evidence.cite_fields_off", "analysis.input_invalid"}) +#: 说同一件事的画布提醒(schema.analysis_issues 对调用别的工具的「调用工具」节点的那条),以及口径取数 SQL 的两条 +#: lint(门禁和画布说的是同一条 SQL、同一个字段) +_REPEATS = frozenset({"evidence.caliber_compute_input", "evidence.cite_fields_off", "analysis.input_invalid", + *CALIBER_SQL_CODES}) def _not_repeated(issues: list[Any], gate: list[Any], spec: GraphSpec) -> list[Any]: @@ -234,6 +250,7 @@ def _not_repeated(issues: list[Any], gate: list[Any], spec: GraphSpec) -> list[A compute = {i.node_id for i in gate if i.code == "governed.caliber_compute_input"} covered = {("evidence.cite_fields_off", nid, "output_schema") for nid in cited} covered |= {("analysis.input_invalid", i.node_id, i.field) for i in gate if i.code == "governed.analysis_input"} + covered |= {(i.code, i.node_id, i.field) for i in gate if i.code in CALIBER_SQL_CODES} for card in spec.nodes: if card.type == NodeType.METRICS and compute: covered |= {("evidence.caliber_compute_input", card.id, f"metrics[{i}].expression") diff --git a/backend/app/engine/schema.py b/backend/app/engine/schema.py index e2e036b0..f321593d 100644 --- a/backend/app/engine/schema.py +++ b/backend/app/engine/schema.py @@ -3,6 +3,7 @@ from __future__ import annotations import math import re from collections import defaultdict +from collections.abc import Mapping from enum import StrEnum from typing import Any, Iterable, Literal @@ -1645,6 +1646,9 @@ def text_parse_issues(spec: GraphSpec) -> list[ValidationIssue]: EVIDENCE_ROLES = ("source", "compute") _REF_HEAD = re.compile(r"^(vars|nodes)\.([\w-]+)") +#: 口径卡上游取数 SQL 的书写问题(data/caliber_lint.py):取整、补 0 应交给口径卡 +CALIBER_SQL_CODES = frozenset({"caliber.round_in_source", "caliber.zero_default"}) + def _caliber_inputs(card: GraphNode, spec: GraphSpec) -> list[tuple[int, GraphNode]]: """口径卡每个指标表达式里读到的节点:(指标下标, 产出它的节点)。""" @@ -1670,6 +1674,40 @@ def _caliber_inputs(card: GraphNode, spec: GraphSpec) -> list[tuple[int, GraphNo return out +def caliber_sql_issues(spec: GraphSpec) -> list[tuple[GraphNode, dict[str, Any]]]: + """口径卡上游取数节点里写坏了的取数 SQL:(取数节点, 问题),每个节点最多一条。 + + 取数 SQL 里的 ROUND、COALESCE(…, 0) 让未取整的查询永远认不成等价(ledger/match 只认规范化后完全相同), + 取整、缺省值应当交给口径卡。定义来自上游的卡(caliber_from)不在本图,跳过;调用工具节点照 ledger/match + 的认法:工具名以 QUERY_PREFIX 开头、SQL 里没有「{{」运行时模板。 + """ + from app.data.caliber_lint import issues_for_sql + from app.tools.datasource import QUERY_PREFIX + + out: list[tuple[GraphNode, dict[str, Any]]] = [] + seen: set[str] = set() + for card in spec.nodes: + if card.type != NodeType.METRICS or card.config.get("caliber_from") not in (None, "", {}): + continue + for _, feeder in _caliber_inputs(card, spec): + if feeder.id in seen: + continue + seen.add(feeder.id) + if feeder.type != NodeType.TOOL: + continue + cfg = feeder.config + args = cfg.get("args") + sql = args.get("sql") if isinstance(args, Mapping) else None + if not str(cfg.get("tool") or "").startswith(QUERY_PREFIX): + continue + if not isinstance(sql, str) or not sql.strip() or "{{" in sql: + continue + found = issues_for_sql(sql) + if found: + out.append((feeder, found[0])) + return out + + def _is_zero(node: Any) -> bool: """常量 0(0、0.0、-0、+0);布尔不算。""" import ast @@ -1745,6 +1783,9 @@ def evidence_issues(spec: GraphSpec) -> list[ValidationIssue]: "无法追溯出处。负责取数的代码节点请将「证据角色」设为「取数」;业务计算请移到口径卡的表达式中", node, f"metrics[{i}].expression", code="evidence.caliber_compute_input") + for feeder, problem in caliber_sql_issues(spec): + add(f"取数节点「{feeder.title}」:{problem['message']}({problem['excerpt']})", + feeder, "args.sql", code=problem["code"]) return out diff --git a/backend/tests/test_caliber_lint.py b/backend/tests/test_caliber_lint.py new file mode 100644 index 00000000..6faaa8ca --- /dev/null +++ b/backend/tests/test_caliber_lint.py @@ -0,0 +1,117 @@ +"""口径取数 SQL 的书写 lint:取整、补 0 交给口径卡,画布提示、发布门禁拦截。 + +- 口径卡上游取数 SQL 里的 ROUND(...)、COALESCE / IFNULL / NVL(..., 0):evidence_issues 在取数节点上给 warning +- 受管发布门禁报 error,已发布档只 warning:和别的受管要求同一个口径 +- 解析不了的 SQL 不抛异常;定义来自上游的卡(caliber_from)不在本图,不查 +""" +from __future__ import annotations + +import pytest + +from app.data.caliber_lint import issues_for_sql +from app.engine.governance import lint_for_publish, publish_issues +from app.engine.schema import GraphSpec, evidence_issues + +ROUND = "caliber.round_in_source" +ZERO = "caliber.zero_default" +CELL = "cell(nodes.fetch, 0, 'gmv')" + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, + "data": {"label": label or nid, "config": config}} + + +def graph(sql: str, *, metrics=None, **card) -> dict: + """input → 取数 → 口径卡。默认两个指标都读同一个取数节点,用来验证同一节点只报一次。""" + default = [{"id": "gmv", "name": "销售额", "unit": "元", "expression": CELL}, + {"id": "orders", "name": "订单数", "unit": "单", "expression": CELL}] + nodes = [ + node("start", "input", "周期"), + node("fetch", "tool", "取数", tool="db_query__shop", args={"sql": sql}), + node("card", "metrics", "周报口径卡", metrics=metrics or default, **card), + ] + return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} + for a, b in zip(nodes, nodes[1:])]} + + +def gate(graph_dict: dict, code: str, level: str = "governed") -> list: + return [i for i in lint_for_publish(GraphSpec.model_validate(graph_dict), level=level).issues + if i.code == code] + + +# -------------------------------------------------------------------------- +# SQL 级检查 +# -------------------------------------------------------------------------- + + +def test_round_in_source_is_detected(): + found = issues_for_sql("SELECT ROUND(SUM(amount), 2) AS gmv FROM orders") + assert [i["code"] for i in found] == [ROUND] + assert found[0]["excerpt"] == "ROUND(SUM(amount), 2)" + assert "永远认不成等价" in found[0]["message"] + + +def test_zero_default_needs_a_literal_zero_last_argument(): + found = issues_for_sql("SELECT COALESCE(SUM(amount), 0) AS gmv FROM orders") + assert [i["code"] for i in found] == [ZERO] + assert "没有数据" in found[0]["message"] + assert [i["code"] for i in issues_for_sql("SELECT COALESCE(SUM(amount), -0) AS gmv FROM orders")] == [ZERO] + assert issues_for_sql("SELECT COALESCE(SUM(amount), SUM(refund)) AS gmv FROM orders") == [] + assert issues_for_sql("SELECT COALESCE(SUM(amount), 1) AS gmv FROM orders") == [] + + +@pytest.mark.parametrize("sql", ["SELECT IFNULL(SUM(amount), 0) AS gmv FROM orders", + "SELECT NVL(SUM(amount), 0) AS gmv FROM orders"]) +def test_ifnull_and_nvl_count_as_zero_default(sql): + assert [i["code"] for i in issues_for_sql(sql)] == [ZERO] + + +def test_unparsable_sql_is_ignored(): + assert issues_for_sql("SELECT (((") == [] + assert issues_for_sql("") == [] + assert issues_for_sql(None) == [] # type: ignore[arg-type] + + +# -------------------------------------------------------------------------- +# 画布校验:落在取数节点上,一个节点只报一次 +# -------------------------------------------------------------------------- + + +def test_evidence_warns_on_the_fetch_node_once(): + spec = GraphSpec.model_validate(graph("SELECT ROUND(SUM(amount), 2) AS gmv FROM orders")) + found = [i for i in evidence_issues(spec) if i.code == ROUND] + assert len(found) == 1 + assert found[0].level == "warning" + assert found[0].node_id == "fetch" + assert found[0].field == "args.sql" + + +def test_evidence_skips_cards_from_upstream(): + ref = {"workflow_id": "lib", "workflow_version": 1, "node_id": "card"} + spec = GraphSpec.model_validate( + graph("SELECT COALESCE(SUM(amount), 0) AS gmv FROM orders", caliber_from=ref)) + assert [i for i in evidence_issues(spec) if i.code in (ROUND, ZERO)] == [] + + +# -------------------------------------------------------------------------- +# 发布门禁:已发布只警告,受管挡住 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("code,sql", [ + (ROUND, "SELECT ROUND(SUM(amount), 2) AS gmv FROM orders"), + (ZERO, "SELECT COALESCE(SUM(amount), 0) AS gmv FROM orders"), +]) +def test_governed_blocks_and_published_warns(code, sql): + graph_dict = graph(sql) + assert [i.level for i in gate(graph_dict, code, "governed")] == ["error"] + assert [i.level for i in gate(graph_dict, code, "published")] == ["warning"] + + +def test_publish_issues_lists_the_problem_once(): + """validate 和门禁说的是同一处:发布时不能一正一反各来一条。""" + spec = GraphSpec.model_validate(graph("SELECT ROUND(SUM(amount), 2) AS gmv FROM orders")) + found = [i for i in publish_issues(spec, level="published") if i.code == ROUND] + assert len(found) == 1 + assert found[0].level == "warning"