"""口径对账:路径二b,差在哪、差多少(docs/design/next-round/02-口径账本.md 做法 6,总方案 X3)。 **差在哪**在比对时就有(ledger/match:explain_sql_diff 的条目,这次 / 口径两边说),不跑查询。**差多少**要点开才跑: 按那一版口径确定性派生一张只读小图(ledger/derive.reconcile_graph,全站唯一的派生图生成器),跑一次探索运行 (来源 reconcile,服务端专用、系统派生、不进使用统计): 入口(口径的入口字段,缺省值是这次的参数)→「这次的取数」ours_q1(执行过的 SQL 原文)→ 口径的取数(模板里保留 {{ input.x }})→ 口径卡(caliber_from 固定那一版)→(多行时)按维度对齐的合并查询 align → 对账卡(这次、口径、差值、 相对差)→ 出口(契约 metrics_from 指向对账卡;不加报告、不调模型) 两边在同一次运行里查,差值在对账卡里算(叙述层无算术,也不在 Python 里算:compare.py 是同一工作流两次运行配对用的)。 面板上每个数都点得开 /evidence/metrics 的出处链。 **参数。** 默认用比对时绑定出来的参数(缺的补缺省值)。参数是启发得来的(绑定在 heuristic 一级配上)、有没绑上的、补完 缺省值表单还不齐、或者里面有遮罩值时,要人先核对:动作里写 needs_check 和原因,发起时要带 confirm 和 inputs(按口径入口 的字段校验,ledger/compose.check_inputs 同一套),界面写「参数按这次的 SQL 推断,请核对」。 **列怎么配。** 只有一行结果时:口径卡上的指标式就是取数第一行某一列(cell(nodes.取数, 0, '列'))的,按那一列在口径取数 结果里的位置配这次结果同一位置的列(等价时已实测列的先后计;不等价的两条 SQL 选择列表一样时位置照样对得上)。比对时 按引用的列投影过的(候选的 projection,ledger/match 的 _projected),按投影之后两边留下的列逐个配(这次多选的列不算)。差异 有兜底说明(fallback:两条 SQL 对不上结构)、列数不同、一个都配不上时不猜,要人选(pairs:{ours: 这次的列, metric: 口径卡 上的指标 id}),这个选择记进对账运行的 origin。 **多行结果**(这次的结果不止一行):被报告引用了的那几行(最多 MAX_ROWS 行),按维度列(不是数的列)经合并查询和口径 取数的结果按键对齐(LEFT JOIN,按引用的先后排),每个被引用的行、每个被引用的列出一组对账指标。两边的列按位置配(列数 不同时不猜,照实拒绝)。维度列是遮罩列、或者这次的结果里合并键不唯一时照实拒绝:前者对齐要用到遮住的值,后者对不准 是哪一行。被引用的列是遮罩列的同样不对账。 只读:派生图只查库、不写;使用者发起时带上 MASK_RESULTS / READ_ONLY(同问数据),搭建者也带 READ_ONLY。对账运行记在 使用者名下,但它的原始记录、事件、运行时的图不给使用者(api/deps.run_events_readable 只给问数据的运行):图里原样是口径 取数节点的 SQL 模板,他经账本和证据接口看结果(展示的 SQL 按遮罩换过)。 """ from __future__ import annotations import json from dataclasses import dataclass, field from typing import Any from sqlalchemy import select from app.db.base import SessionLocal from app.db.models import Run, Workflow ORIGIN = "reconcile" #: 被拒的机器码 REFUSED = "ledger_reconcile_refused" CHECK_INPUTS = "ledger_reconcile_check_inputs" NEEDS_PAIRS = "ledger_reconcile_needs_pairs" FORBIDDEN = "ledger_reconcile_forbidden" #: 多行对账最多几行、几列(每行每列出四个对账指标) MAX_ROWS = 10 MAX_COLUMNS = 5 #: 给人看的说法 NOT_FOUND = "运行记录不存在" NO_ACCESS = "没有权限查看这次运行" NOT_MATCHED = "这次运行还没有比对口径,比对完才能对账" NO_QUERY = "这次运行的比对记录里没有这条查询" NOT_RELATED = "这条查询没有和「{name}」v{version} 比出差异(只有相关口径才对账)" NO_CARD = "「{name}」v{version} 里没有读这个取数节点的口径卡,对不了账" SNAPSHOT = "这条查询的快照读不出来(文件缺失或与哈希不一致),对不了账" CHECK_REASON = "参数按这次的 SQL 推断,请核对" CHECK_MASKED = "参数里有遮罩值,请核对参数" CHECK_INCOMPLETE = "按这次的 SQL 没能推断出全部参数,请补全并核对" CONFIRM_FIRST = "参数按这次的 SQL 推断,发起前请先核对参数" PAIRS_FALLBACK = "两条 SQL 的结构对不上,认不出这次的哪一列对应口径的哪个指标,请选择要对照的指标" PAIRS_NONE = "认不出这次的哪一列对应口径的哪个指标,请选择要对照的指标" PAIR_COLUMN = "这次的结果里没有列「{column}」" PAIR_MASKED = "列「{column}」设置了遮罩,不能对账" PAIR_METRIC = "口径卡「{card}」上没有指标「{metric}」" PAIRS_SHAPE = "要对照的指标需要写成一组「这次的列 → 口径卡上的指标」的对照" ROWS_NONE = "报告里没有引用这条查询的格,不知道要对哪几行" ROWS_NO_KEY = "这次的结果有多行,但没有可以用来对齐的维度列(不是数的列)" ROWS_MASKED_KEY = "用来对齐的维度列「{column}」设置了遮罩,不能对账" ROWS_DUPLICATE = "这次的结果里按维度列({columns})有重复的行,对不准是哪一行" ROWS_COLUMNS = "这次的结果和口径的取数列数不同({ours} 列对 {theirs} 列),按位置对不上" ROWS_CALIBER_COLUMNS = "认不出口径取数的结果列,按位置对不上" class ReconcileRefused(Exception): """不能对账。extra 是给界面的几样(并进 409 的返回体):认不出列怎么配时的 options(可选的列、口径卡上的指标),界面 照它画「选要对照的指标」那一步,不自己去猜有哪些列。""" def __init__(self, status_code: int, message: str, code: str = REFUSED, *, extra: dict[str, Any] | None = None) -> None: super().__init__(message) self.status_code = status_code self.message = message self.code = code self.extra = dict(extra or {}) # ========================================================================== # 动作:GET 账本里能对的账 # ========================================================================== def _needs_check(cand: dict[str, Any], fields: list[dict[str, Any]], hide: Any) -> str | None: """参数要不要人先核对:要的回原因(见模块说明),不要的为 None。fields 是比对工件里记的入口字段(name、required、 has_default)。""" binding = cand.get("binding") or {} if hide is not None and any(hide(v) for v in binding.values()): return CHECK_MASKED if cand.get("unbound") or not binding and any(f.get("name") for f in fields): return CHECK_INCOMPLETE if any(f.get("required") and not f.get("has_default") and f.get("name") not in binding for f in fields): return CHECK_INCOMPLETE if cand.get("via") in (None, "heuristic"): return CHECK_REASON return None def actions_of(content: dict[str, Any], names: dict[str, str], hide: Any = None) -> list[dict[str, Any]]: """比对工件里每条有相关口径的查询、每份相关口径一项「看差多少」:{kind: reconcile, query, workflow_id, version, node_id, needs_check, check_reason?, binding(遮过的), fields}。界面据此摆按钮、要不要先核对参数;fields 是那一版口径的入口字段 (名字、必填、有没有缺省值,比对工件里记的那一份),核对参数的表单照它画(缺的参数也写得出是哪一项)。""" from app.ledger.labels import LEVEL_TEXT, binding_view hide = hide or (lambda _value: False) fields = {c.get("caliber"): c.get("fields") or [] for c in content.get("calibers") or []} out: list[dict[str, Any]] = [] for query in content.get("queries") or []: if query.get("status") not in ("related",): continue for cand in query.get("candidates") or []: if cand.get("status") != "related" or cand.get("comparable") is False: continue why = _needs_check(cand, list(fields.get(cand.get("caliber")) or []), hide) item = {"kind": "reconcile", "query": query.get("artifact"), "caliber": cand.get("caliber"), "workflow_id": cand.get("workflow_id"), "version": cand.get("version"), "node_id": cand.get("node_id"), "workflow_name": names.get(str(cand.get("workflow_id") or "")), "level_text": LEVEL_TEXT.get(str(cand.get("level") or ""), ""), "needs_check": why is not None, "binding": binding_view(cand.get("binding"), hide), "fallback": bool((cand.get("diff") or {}).get("fallback")), "fields": [{"name": str(f.get("name")), "required": bool(f.get("required")), "has_default": bool(f.get("has_default"))} for f in fields.get(cand.get("caliber")) or [] if f.get("name")]} if why is not None: item["check_reason"] = why out.append(item) return out async def reconciled_runs(session: Any, run_id: str) -> list[Run]: """对这次运行对过的账(origin.source_run 是它),新的在前。""" from app.db.portable import json_eq rows = (await session.execute(select(Run).where( Run.origin_kind == ORIGIN, json_eq(Run.origin, "source_run", run_id)) .order_by(Run.created_at.desc(), Run.id.desc()))).scalars() return list(rows) #: 对账卡上一组对照的四个指标:id 的前缀(tail_nodes 的 _four)和名字的后缀 _FOUR = (("ours", "这次"), ("caliber", "口径"), ("diff", "差值"), ("rel", "相对差")) _PREFIX = {"ours": "ours_", "caliber": "cal_", "diff": "diff_", "rel": "rel_"} def reconcile_groups(graph: Any) -> list[dict[str, str]]: """对账运行的图里,对账卡上的每一组对照:{label, ours, caliber, diff, rel}(后四个是指标 id)。按卡上的先后;名字 是「<这一组的说法> · 这次」那样写的,label 去掉后缀。认不出的(不是派生的对账图)为空。""" from app.ledger.derive import RECONCILE_CARD nodes = (graph or {}).get("nodes") if isinstance(graph, dict) else None card = next((n for n in nodes or [] if isinstance(n, dict) and n.get("id") == RECONCILE_CARD and n.get("type") == "metrics"), None) metrics = (((card or {}).get("data") or {}).get("config") or {}).get("metrics") or [] by_id = {str(m.get("id")): m for m in metrics if isinstance(m, dict) and m.get("id")} out: list[dict[str, str]] = [] for mid, metric in by_id.items(): if not mid.startswith(_PREFIX["ours"]): continue n = mid[len(_PREFIX["ours"]):] ids = {role: f"{_PREFIX[role]}{n}" for role, _ in _FOUR} if not all(i in by_id for i in ids.values()): continue name = str(metric.get("name") or "") label = name[:-len(" · 这次")] if name.endswith(" · 这次") else name out.append({"label": label, **ids}) return out def reconciled_view(run: Run) -> dict[str, Any]: """GET 账本 reconciled 里的一项:哪一次对账运行、状态、对的哪条查询和哪份口径、参数从哪来;card 和 groups 是对账卡和 每一组对照的四个指标(界面按 /evidence/metrics/{card}/{指标} 取值、点开出处,不按命名规则自己拼)。""" from app.ledger.derive import RECONCILE_CARD origin = run.origin if isinstance(run.origin, dict) else {} return {"run_id": run.id, "status": run.status, "query": origin.get("query"), "caliber": origin.get("caliber"), "workflow_id": origin.get("workflow_id"), "version": origin.get("version"), "inputs_from": origin.get("inputs_from"), "rows": origin.get("rows"), "card": RECONCILE_CARD, "groups": reconcile_groups(run.graph), "created_at": run.created_at.isoformat() if run.created_at else None} # ========================================================================== # 派生图 # ========================================================================== @dataclass class Pair: """一组对照:这次结果的哪一列,对口径卡上的哪个指标(metric)或口径取数结果的哪一列(column)。""" ours: str metric: str | None = None column: str | None = None #: 口径卡上那个指标的位置(metric 配对时) index: int | None = None name: str = "" unit: str = "" decimals: Any = None def describe(self) -> dict[str, Any]: return {"ours": self.ours, **({"metric": self.metric} if self.metric else {}), **({"caliber": self.column} if self.column else {})} @dataclass class Plan: """一次对账要派生的东西(纯计算的结果,测试锁住确定性)。""" pairs: list[Pair] rows: list[int] | None = None keys: list[tuple[str, str]] = field(default_factory=list) key_values: list[tuple[Any, ...]] = field(default_factory=list) def _lit(value: Any) -> str: """一个值写进合并查询(SQLite)的字面量。""" if value is None: return "NULL" if isinstance(value, bool): return "1" if value else "0" if isinstance(value, (int, float)): return repr(value) return "'" + str(value).replace("'", "''") + "'" def _ident(name: str) -> str: return '"' + str(name).replace('"', '""') + '"' def _equals(alias: str, column: str, value: Any) -> str: return f"{alias}.{_ident(column)} IS NULL" if value is None else f"{alias}.{_ident(column)} = {_lit(value)}" def align_sql(keys: list[tuple[str, str]], pairs: list[Pair], values: list[tuple[Any, ...]]) -> str: """多行对账的合并查询:这次(o)LEFT JOIN 口径(c)按维度列对齐,只留被引用的那几行,按引用的先后排。""" select_ = [f"o.{_ident(ours)} AS {_ident(ours)}" for ours, _ in keys] select_ += [f"o.{_ident(p.ours)} AS {_ident(f'ours_{j}')}, c.{_ident(p.column or '')} AS {_ident(f'cal_{j}')}" for j, p in enumerate(pairs)] on = " AND ".join(f"o.{_ident(ours)} = c.{_ident(theirs)}" for ours, theirs in keys) rows = ["(" + " AND ".join(_equals("o", ours, v) for (ours, _), v in zip(keys, row)) + ")" for row in values] order = " ".join(f"WHEN {cond} THEN {i}" for i, cond in enumerate(rows)) return (f"SELECT {', '.join(select_)} FROM o LEFT JOIN c ON {on} WHERE {' OR '.join(rows)} " f"ORDER BY CASE {order} END") def _metric(mid: str, name: str, expression: str, *, unit: str = "", decimals: Any = None, fmt: str | None = None) -> dict[str, Any]: out: dict[str, Any] = {"id": mid, "name": name, "unit": unit, "expression": expression} if decimals is not None: out["decimals"] = decimals if fmt: out["format"] = fmt return out def _four(n: str, label: str, ours: str, theirs: str, pair: Pair) -> list[dict[str, Any]]: """一组对照的四个对账指标:这次、口径、差值(这次 − 口径)、相对差(差值 ÷ 口径,口径是 0 时没有值)。""" diff = f"({ours}) - ({theirs})" return [ _metric(f"ours_{n}", f"{label} · 这次", ours, unit=pair.unit, decimals=pair.decimals), _metric(f"cal_{n}", f"{label} · 口径", theirs, unit=pair.unit, decimals=pair.decimals), _metric(f"diff_{n}", f"{label} · 差值", diff, unit=pair.unit, decimals=pair.decimals), _metric(f"rel_{n}", f"{label} · 相对差", f"({diff}) / ({theirs}) if ({theirs}) else None", decimals=4, fmt="percent_of_ratio"), ] def tail_nodes(plan: Plan, *, card_id: str, fetch_id: str) -> list[dict[str, Any]]: """派生图的 tail:(多行时)对齐的合并查询 + 对账卡。card_id、fetch_id 是图里口径卡、口径取数节点的 id。""" from app.ledger.derive import ALIGN_ID, OURS_ID, RECONCILE_CARD, _node metrics: list[dict[str, Any]] = [] nodes: list[dict[str, Any]] = [] if plan.rows is None: for j, pair in enumerate(plan.pairs): ours = f"cell(nodes.{OURS_ID}, 0, {pair.ours!r})" theirs = (f"nodes.{card_id}.metrics[{pair.index}].value" if pair.metric is not None else f"cell(nodes.{fetch_id}, 0, {pair.column!r})") metrics += _four(str(j), pair.name or pair.ours, ours, theirs, pair) else: nodes.append(_node(ALIGN_ID, "merge", "按维度对齐", inputs={"o": OURS_ID, "c": fetch_id}, sql=align_sql(plan.keys, plan.pairs, plan.key_values))) for i, row in enumerate(plan.key_values): where = ",".join(f"{ours}={v}" for (ours, _), v in zip(plan.keys, row)) # 先核对第 i 行就是这一组的维度:重跑时这次的结果缺了一行、某个维度多出一行(口径那边重复),合并结果会错位, # 按位置取到的是别的维度的数。对不上的记空值(对账卡 on_missing=null),不拿别的维度的数冒充 guard = _row_guard(i, plan.keys, row) for j, pair in enumerate(plan.pairs): ours = f"cell(nodes.{ALIGN_ID}, {i}, 'ours_{j}')" theirs = f"cell(nodes.{ALIGN_ID}, {i}, 'cal_{j}')" if guard: ours, theirs = f"({ours} if {guard} else None)", f"({theirs} if {guard} else None)" metrics += _four(f"{i}_{j}", f"{where} · {pair.name or pair.ours}", ours, theirs, pair) nodes.append(_node(RECONCILE_CARD, "metrics", "对账卡", caliber="口径对账", caliber_version="v1", on_missing="null", metrics=metrics)) return nodes def _row_guard(i: int, keys: list[tuple[str, str]], values: tuple[Any, ...]) -> str: """对齐结果第 i 行是不是这一组(维度列的值都对得上)的条件式;维度值是空值的那一列不核对(表达式里没有空值的写法)。""" import ast from app.engine.expressions import _literal parts = [f"cell(nodes.{_align_id()}, {i}, {ours!r}) == {ast.unparse(_literal(v))}" for (ours, _), v in zip(keys, values) if v is not None and not isinstance(v, (list, dict))] return " and ".join(parts) def _align_id() -> str: from app.ledger.derive import ALIGN_ID return ALIGN_ID def _numeric(rows: list[Any], index: int) -> bool: values = [r[index] for r in rows if isinstance(r, (list, tuple)) and index < len(r) and r[index] is not None] return bool(values) and all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in values) def _plain_cell(expression: str, fetch: str) -> str | None: """指标式就是取数第一行某一列(cell(nodes., 0, '列'))时交回列名,别的写法为 None。""" import ast from app.engine.expressions import cell_parts parts = cell_parts(expression.strip()) if parts is None or parts[0] != f"nodes.{fetch}" or parts[1].strip() != "0": return None try: column = ast.literal_eval(parts[2]) except (ValueError, SyntaxError): return None return column if isinstance(column, str) else None def plan_pairs(*, ours_columns: list[str], ours_rows: list[Any], hidden: set[str], caliber_columns: list[str] | None, metrics: list[dict[str, Any]], fetch: str, card: str, fallback: bool, chosen: Any, referenced: list[tuple[int, str]], projected: Any = None, cells: Any = None, caliber: str | None = None) -> Plan: """配列、(多行时)定行和键:交回 Plan,配不了的抛 ReconcileRefused(见模块说明)。projected 是比对候选的 projection (按引用的列投影过的:{ours: [这次留下的列], caliber: [口径留下的列]},按位置一一对应)。cells 是这条查询逐格比的结论 (ledger/match._cells),caliber 是这份口径的键:要人选对照时按它预选(_suggested)。纯计算。""" def check_column(column: str) -> None: if column not in ours_columns: raise ReconcileRefused(422, PAIR_COLUMN.format(column=column)) if column.lower() in hidden: raise ReconcileRefused(409, PAIR_MASKED.format(column=column)) by_id = {str(m.get("id")): (i, m) for i, m in enumerate(metrics)} multi = len(ours_rows) > 1 # 要人选对照时给界面的选项:这次结果里是数、没遮罩的列,口径卡上的指标(id、名字、单位) numeric = [c for i, c in enumerate(ours_columns) if c.lower() not in hidden and _numeric(ours_rows, i)] options = {"options": {"columns": numeric, "metrics": [{"id": str(m.get("id")), "name": str(m.get("name") or m.get("id")), "unit": str(m.get("unit") or "")} for m in metrics if m.get("id")], "suggested": _suggested(cells, caliber, fetch, numeric, metrics)}} if multi: return _plan_rows(ours_columns, ours_rows, hidden, caliber_columns, referenced, check_column) if chosen is not None: if not isinstance(chosen, list) or not chosen or not all(isinstance(p, dict) for p in chosen): raise ReconcileRefused(422, PAIRS_SHAPE, NEEDS_PAIRS, extra=options) pairs = [] for item in chosen: ours, metric = str(item.get("ours") or ""), str(item.get("metric") or "") check_column(ours) if metric not in by_id: raise ReconcileRefused(422, PAIR_METRIC.format(card=card, metric=metric)) index, m = by_id[metric] pairs.append(Pair(ours=ours, metric=metric, index=index, name=str(m.get("name") or metric), unit=str(m.get("unit") or ""), decimals=m.get("decimals"))) return Plan(pairs=pairs) if fallback: raise ReconcileRefused(409, PAIRS_FALLBACK, NEEDS_PAIRS, extra=options) pairs = [] kept = _projected_pairs(projected, ours_columns) if kept is not None: # 按投影之后留下的列配:这次多选的列不算,口径一侧只看投影留下的那几列(列名不分大小写) for index, m in enumerate(metrics): column = _plain_cell(str(m.get("expression") or ""), fetch) ours = next((o for o, c in kept if column is not None and c.lower() == column.lower()), None) if ours is None or ours.lower() in hidden: continue pairs.append(Pair(ours=ours, metric=str(m.get("id")), index=index, name=str(m.get("name") or m.get("id")), unit=str(m.get("unit") or ""), decimals=m.get("decimals"))) elif caliber_columns is not None and len(caliber_columns) == len(ours_columns): for index, m in enumerate(metrics): column = _plain_cell(str(m.get("expression") or ""), fetch) if column is None or column not in caliber_columns: continue ours = ours_columns[caliber_columns.index(column)] if ours.lower() in hidden: continue pairs.append(Pair(ours=ours, metric=str(m.get("id")), index=index, name=str(m.get("name") or m.get("id")), unit=str(m.get("unit") or ""), decimals=m.get("decimals"))) if not pairs: raise ReconcileRefused(409, PAIRS_NONE, NEEDS_PAIRS, extra=options) return Plan(pairs=pairs) def _suggested(cells: Any, caliber: str | None, fetch: str, columns: list[str], metrics: list[dict[str, Any]]) -> list[dict[str, str]]: """要人选对照时预选哪几对:[{ours, metric}]。按逐格比的结论(ledger/match._cells)——这次结果的这一列单独比,和这份口径 取数节点的某一列等价(逐格证明了,projection.caliber 是那一列),口径卡上只读这一列这一格的指标就是它最可能的对照。只认 证明了的:只是「相关」的那一格比到的是第一个试的列,不是最像的,不替人猜;一个口径列对上几个指标的也不预选。界面照它 预选、人可以改。""" from app.engine.expressions import ExpressionError, cell_parts, leaf_refs, parse_expression if not isinstance(cells, dict) or not caliber: return [] reads: dict[str, list[str]] = {} for m in metrics: try: leaves = leaf_refs(parse_expression(str(m.get("expression") or ""))[0]) except ExpressionError: continue cols = {_plain_cell(str(leaf), fetch) for leaf in leaves if cell_parts(str(leaf)) is not None} if len(leaves) == 1 and len(cols) == 1 and None not in cols and m.get("id"): reads.setdefault(str(next(iter(cols))).lower(), []).append(str(m["id"])) out: list[dict[str, str]] = [] for col in columns: cell = cells.get(col) cands = cell.get("candidates") if isinstance(cell, dict) else None proven = next((c for c in cands or [] if isinstance(c, dict) and c.get("status") == "proven" and c.get("caliber") == caliber and c.get("node_id") == fetch), None) theirs = ((proven or {}).get("projection") or {}).get("caliber") if proven else None if not isinstance(theirs, list) or len(theirs) != 1 or not isinstance(theirs[0], str): continue hits = reads.get(theirs[0].lower()) or [] if len(hits) == 1: out.append({"ours": col, "metric": hits[0]}) return out def _projected_pairs(projected: Any, ours_columns: list[str]) -> list[tuple[str, str]] | None: """比对候选的 projection → [(这次的列, 口径的列)],按位置一一对应;不是投影过的、两边列数不一样、这次的列不在结果里的 为 None(照旧按整条的位置配)。""" if not isinstance(projected, dict): return None ours, theirs = projected.get("ours"), projected.get("caliber") if not isinstance(ours, list) or not isinstance(theirs, list) or not ours or len(ours) != len(theirs): return None if any(not isinstance(o, str) or o not in ours_columns or not isinstance(c, str) for o, c in zip(ours, theirs)): return None return list(zip(ours, theirs)) def _plan_rows(ours_columns: list[str], rows: list[Any], hidden: set[str], caliber_columns: list[str] | None, referenced: list[tuple[int, str]], check_column: Any) -> Plan: if caliber_columns is None: raise ReconcileRefused(409, ROWS_CALIBER_COLUMNS) if len(caliber_columns) != len(ours_columns): raise ReconcileRefused(409, ROWS_COLUMNS.format(ours=len(ours_columns), theirs=len(caliber_columns))) keys = [i for i in range(len(ours_columns)) if not _numeric(rows, i)] if not keys: raise ReconcileRefused(409, ROWS_NO_KEY) if masked := [ours_columns[i] for i in keys if ours_columns[i].lower() in hidden]: raise ReconcileRefused(409, ROWS_MASKED_KEY.format(column=masked[0])) tuples = [tuple(r[i] if isinstance(r, (list, tuple)) and i < len(r) else None for i in keys) for r in rows] if len(set(map(json.dumps, [list(t) for t in tuples]))) != len(tuples): raise ReconcileRefused(409, ROWS_DUPLICATE.format(columns="、".join(ours_columns[i] for i in keys))) wanted_rows = list(dict.fromkeys(r for r, _ in referenced if isinstance(r, int) and 0 <= r < len(rows)))[:MAX_ROWS] columns = list(dict.fromkeys(c for _, c in referenced if c in ours_columns and ours_columns.index(c) not in keys)) if not wanted_rows or not columns: raise ReconcileRefused(409, ROWS_NONE) for column in columns: check_column(column) pairs = [Pair(ours=c, column=caliber_columns[ours_columns.index(c)], name=c) for c in columns[:MAX_COLUMNS]] return Plan(pairs=pairs, rows=wanted_rows, keys=[(ours_columns[i], caliber_columns[i]) for i in keys], key_values=[tuples[r] for r in wanted_rows]) def _referenced(sealed: Any, artifact: str) -> list[tuple[int, str]]: """报告里引用了这条查询的哪几格:[(行, 列)],按正文的先后(单元格、表格里的格)。""" from app.api.evidence import _reports from app.engine.evidence import iter_units out: list[tuple[int, str]] = [] for report in _reports(sealed): doc = report.doc if report.hash_ok is True else None if not isinstance(doc, dict): continue catalog = doc.get("catalog") if isinstance(doc.get("catalog"), dict) else {} for _, unit in iter_units(doc): for seg in unit.get("segments") or []: cite = seg.get("cite") if isinstance(seg, dict) and isinstance(seg.get("cite"), dict) else {} entry = catalog.get(cite.get("alias")) if isinstance(catalog.get(cite.get("alias")), dict) else {} locator = cite.get("locator") if isinstance(cite.get("locator"), dict) else {} if cite.get("kind") == "cell" and entry.get("artifact") == artifact \ and isinstance(locator.get("row"), int) and isinstance(locator.get("column"), str): out.append((locator["row"], locator["column"])) return out def _caliber_columns(template: str, inputs: dict[str, Any], fields: list[dict[str, Any]], dialect: str | None, schema: dict[str, Any] | None) -> list[str] | None: """口径取数按这次的参数渲染之后选出来的列名(按位置);认不出的为 None。""" from app.data.sql_semantics import _read, _result_columns, _schema_info, render_bound try: rendered = render_bound(template, inputs, fields) return _result_columns(rendered, _read(dialect), _schema_info(schema)) except Exception: # noqa: BLE001 - 认不出就按认不出说 return None # ========================================================================== # 发起 # ========================================================================== @dataclass class Request: query: str workflow_id: str version: int inputs: dict[str, Any] | None = None confirm: bool = False pairs: Any = None async def reconcile(run_id: str, req: Request, *, principal: Any) -> Run: """对这次运行的一条查询和一份相关口径对一次账:派生只读图、发起探索运行(来源 reconcile)。不能对的抛 ReconcileRefused。""" from app.api.deps import can_read_run from app.api.evidence import _Masks, _sealed from app.core.artifact_store import load from app.engine.launch import LaunchRefused, launch_run from app.ledger import compose, derive, store from app.ledger.issue import _conversation_of from app.ledger.labels import caliber_key from app.ledger.match import _frozen from app.ledger.view import _hider from app.tools.datasource import MASK_RESULTS_OPTION, QUERY_PREFIX, READ_ONLY_OPTION async with SessionLocal() as session: run = await session.get(Run, run_id) if run is None: raise ReconcileRefused(404, NOT_FOUND) if principal.mode != "off" and not await can_read_run(principal, run): raise ReconcileRefused(403, NO_ACCESS, FORBIDDEN) recorded = await store.latest(run_id) if recorded is None or recorded.content is None: raise ReconcileRefused(409, NOT_MATCHED) content = recorded.content key = caliber_key(req.workflow_id, int(req.version)) query = next((q for q in content.get("queries") or [] if q.get("artifact") == req.query), None) if query is None: raise ReconcileRefused(404, NO_QUERY) async with SessionLocal() as session: workflow = await session.get(Workflow, req.workflow_id) name = workflow.name if workflow is not None else req.workflow_id cand = next((c for c in query.get("candidates") or [] if c.get("caliber") == key and c.get("status") == "related"), None) if cand is None: raise ReconcileRefused(409, NOT_RELATED.format(name=name, version=req.version)) summary = next((c for c in content.get("calibers") or [] if c.get("caliber") == key), {}) pinned = None async with SessionLocal() as session: for card_id in summary.get("cards") or []: try: candidate = await derive.load_pinned(session, req.workflow_id, int(req.version), str(card_id)) except derive.NotDerivable: continue if any(node == cand.get("node_id") for node, _, _ in candidate.reads): pinned = candidate break if pinned is None: raise ReconcileRefused(409, NO_CARD.format(name=name, version=req.version)) # 参数:比对时绑定出来的(补缺省值),要核对的要人带着核对过的参数再来 sealed = await _sealed(run_id) hider = await _hider(sealed) why = _needs_check(cand, list(summary.get("fields") or []), hider.hide) if req.confirm and req.inputs is not None: try: inputs = compose.check_inputs(pinned, req.inputs) except compose.ComposeRefused as e: raise ReconcileRefused(422, e.message, CHECK_INPUTS) from e inputs_from = "checked" elif why is not None: raise ReconcileRefused(409, CONFIRM_FIRST if why == CHECK_REASON else why, CHECK_INPUTS) else: try: inputs = compose.check_inputs(pinned, dict(cand.get("binding") or {})) except compose.ComposeRefused as e: raise ReconcileRefused(409, f"{CHECK_INCOMPLETE}:{e.message}", CHECK_INPUTS) from e inputs_from = "binding" # 这次的那条查询:快照、遮罩列、被引用的格 try: snap = load(req.query) except ValueError: snap = None if not isinstance(snap, dict) or not isinstance(snap.get("sql"), str) or req.query not in sealed.queries: raise ReconcileRefused(409, SNAPSHOT) source = str(snap.get("source") or query.get("source") or "") columns = [str(c) for c in snap.get("columns") or []] rows = snap.get("rows") if isinstance(snap.get("rows"), list) else [] masks = _Masks(sealed) hidden = set((await masks.sets(req.query, source, snap)).hidden) kind = await masks.kind(source) schema, _ = _frozen(snap.get("schema_artifact")) fetch = next(n for n in pinned.fetches if n["id"] == cand.get("node_id")) template = str(((fetch.get("data") or {}).get("config") or {}).get("args", {}).get("sql") or "") plan = plan_pairs(ours_columns=columns, ours_rows=rows, hidden=hidden, caliber_columns=_caliber_columns(template, inputs, list(pinned.entry_fields), kind, schema), metrics=pinned.metrics, fetch=str(fetch["id"]), card=pinned.card_title, fallback=bool((cand.get("diff") or {}).get("fallback")), chosen=req.pairs, referenced=_referenced(sealed, req.query), projected=cand.get("projection"), cells=query.get("cells"), caliber=key) graph = build(pinned, sql=snap["sql"], tool=f"{QUERY_PREFIX}{source}", plan=plan, inputs=inputs, fetch=str(fetch["id"])) async with SessionLocal() as session: origin = {"source_run": run_id, **await _conversation_of(session, run), "query": req.query, "caliber": key, "workflow_id": req.workflow_id, "version": int(req.version), "inputs_from": inputs_from, "binding_via": cand.get("via"), "pairs": [p.describe() for p in plan.pairs], **({"rows": plan.rows} if plan.rows is not None else {}), **({"pairs_chosen": True} if req.pairs is not None else {}), "derive": graph.origin} options = {READ_ONLY_OPTION: True} if principal.mode != "off" and not principal.at_least("builder"): options[MASK_RESULTS_OPTION] = True try: return await launch_run(session, graph=graph.graph, input_payload=inputs, run_class="exploratory", actor=principal.username, origin_kind=ORIGIN, origin=origin, options=options, workflow_name=f"口径对账:{name} v{req.version}") except LaunchRefused as e: raise ReconcileRefused(e.status_code, e.message, e.code or REFUSED) from e def build(pinned: Any, *, sql: str, tool: str, plan: Plan, inputs: dict[str, Any], fetch: str) -> Any: """按 Plan 派生对账图(ledger/derive.reconcile_graph)。fetch 是这条查询比的那个口径取数节点(那一版里的 id)。 纯计算,同样的输入同一张图。""" from app.ledger import derive prefix = derive.reconcile_prefix(pinned) tail = tail_nodes(plan, card_id=f"{prefix}{pinned.node_id}", fetch_id=f"{prefix}{fetch}") return derive.reconcile_graph(pinned, derive.ours_block(sql, tool), tail, inputs=inputs) __all__ = ["CHECK_INPUTS", "MAX_COLUMNS", "MAX_ROWS", "NEEDS_PAIRS", "ORIGIN", "REFUSED", "Pair", "Plan", "ReconcileRefused", "Request", "actions_of", "align_sql", "build", "plan_pairs", "reconcile", "reconcile_groups", "reconciled_runs", "reconciled_view", "tail_nodes"]