From 86e52192a9c231b9e4e4fa045ca5ba7e82cb22d9 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 08:49:52 +0800 Subject: [PATCH 1/8] =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=AF=B9=E6=AF=94?= =?UTF-8?q?=E5=90=8E=E7=AB=AF=EF=BC=88=E5=8F=98=E5=8C=96=E4=B8=89=E5=88=86?= =?UTF-8?q?=E6=B3=95=EF=BC=89=EF=BC=9A=E4=B8=A4=E6=AC=A1=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E6=8C=89=E6=8C=87=E6=A0=87=E6=8B=86=E6=B8=85=E6=98=AF=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=8F=98=E4=BA=86=E3=80=81=E5=8F=A3=E5=BE=84=E5=8F=98?= =?UTF-8?q?=E4=BA=86=E8=BF=98=E6=98=AF=E6=A8=A1=E6=9D=BF=20/=20SQL=20?= =?UTF-8?q?=E5=8F=98=E4=BA=86=EF=BC=8C=E6=95=B0=E6=8D=AE=E5=8F=98=E5=8C=96?= =?UTF-8?q?=E6=8C=89=E7=BB=B4=E5=BA=A6=E5=BD=92=E5=9B=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /api/runs/{id}/compare/base 找默认基准：定时任务发起的先找同一定时任务更早结束的一次，否则同一工作流、 同一运行类别；只挑封存过的终态运行，找不到 404。GET /api/runs/{id}/compare?base= 对比两次运行：两次都要 已结束、封存且核对一致（照搬证据接口的封存判定），否则 409；不是同一个工作流、同一次运行 422。只读，不追加 事件、不落工件。 engine/compare.py 是纯函数： - 查询按「节点 + 第几次 + 工具」配对，配不上再按规范化 SQL 配；normalize_sql 判 same / params_only / changed，结构变了的用 diff_facets 写出差在哪（时间范围排在最后）；核查类查询不参与 - 口径卡同一节点按指标 id 配对，表达式、口径版本、小数位、单位、显示格式不同算口径变了 - 每个指标按 added/removed → 口径 → SQL 或上游配置 → 说不清 → 只参数 → 数据 的次序分类；输入落在快照上 的直接比两边快照的 SQL，不落在快照上的看写它的节点和画布上游 - 口径变了时严格拆分：旧表达式配这次的输入用推演重算的还原和求值算 f_old(x_new)，十进制精确相减，两效应之和 逐位等于总变化；SQL 也变了、用到遮罩列、单位不同、旧叶子没记录的不拆并说明 - 维度归因按维度键配对两边的行、取变化最大的 10 个；度量之和等于输入的值（含合计行那一格）才给占比；截断、 度量列遮罩照实写；遮罩列、日期列、每期都不同的列不当维度 --- backend/app/api/compare.py | 167 +++- backend/app/engine/compare.py | 1152 ++++++++++++++++++++++++ backend/tests/fixtures/compare_runs.py | 211 +++++ backend/tests/test_compare_api.py | 387 ++++++++ backend/tests/test_compare_engine.py | 558 ++++++++++++ 5 files changed, 2472 insertions(+), 3 deletions(-) create mode 100644 backend/app/engine/compare.py create mode 100644 backend/tests/fixtures/compare_runs.py create mode 100644 backend/tests/test_compare_api.py create mode 100644 backend/tests/test_compare_engine.py diff --git a/backend/app/api/compare.py b/backend/app/api/compare.py index 582d926c..5b648c45 100644 --- a/backend/app/api/compare.py +++ b/backend/app/api/compare.py @@ -1,10 +1,171 @@ -"""运行对比：两次运行按「变化三分法」对照。 +"""运行对比：同一个工作流的两次运行按「变化三分法」对照——数据变了、口径变了、还是模板 / SQL 变了（engine/compare.py）。 -目前只登记了路由前缀，接口由运行对比的工作包补上。路径挂在 /api/runs/{run_id}/… 下面； -这个路由登记在 runs.router 之前，以后加 /api/runs/compare 这类固定路径也不会被 /{run_id} 吃掉。 +GET /api/runs/{run_id}/compare/base：默认拿哪一次来比，{base_run_id, reason, base}。这次运行是定时任务发起的，取同一 +定时任务、结束得比它早的最近一次终态运行（reason: same_schedule）；不是、或者同一任务还没有更早的，取同一工作流、 +同一运行类别、结束得比它早的最近一次终态运行（reason: same_workflow）。只挑封存过的：没封存的拿来也比不了。 +找不到 404。 + +GET /api/runs/{run_id}/compare?base={base_id}：{target, base, summary, template, metrics, queries, notes}，每个字段 +的形状见 engine/compare.py。两次运行都必须已结束、封存且封存核对一致，封存判定照搬证据接口（api/evidence._sealed）： +没封存的运行事件还会变，封存核对不一致的事件被改过，拿它们比出来的结论都不作数，回 409。不是同一个工作流、或者 +是同一次运行，回 422。 + +只读：两次运行都不追加事件、不落工件、不改运行记录；结果只交回给发请求的人。只认各自封存范围内的工件（证据接口的 +白名单，加上封存的节点产出和指标集里写着的完整值，同推演），遮罩列（查询当时记下的加数据源现在设的）的值不出现在 +结果里。读工件、比 SQL、重算都在线程里做：快照大的时候不挡住同一进程里别的请求。 + +路径挂在 /api/runs/{run_id}/… 下面；这个路由登记在 runs.router 之前，以后加 /api/runs/compare 这类固定路径也不会 +被 /{run_id} 吃掉。定时任务结束后自动对比（期 4）直接调 default_base、compare_runs。 """ from __future__ import annotations +import asyncio +from collections.abc import Callable +from typing import Any + from fastapi import APIRouter +from sqlalchemy import select + +from app.api.coded import CodedHTTPException +from app.api.evidence import _TERMINAL, RUN_NOT_FOUND, _Masks, _Sealed, _sealed, _sealed_cards +from app.api.whatif import _finished_before, _loader, _mask_of, _outputs_before, _returned +from app.db.base import SessionLocal +from app.db.models import DataSource, Run +from app.engine import compare as engine +from app.engine import recompute router = APIRouter(prefix="/api/runs", tags=["compare"]) + +BAD_REQUEST = "compare_bad_request" +MISMATCH = "compare_mismatch" +UNSEALED = "compare_unsealed" +SEAL_BROKEN = "compare_seal_broken" +NO_BASE = "compare_no_base" + +BAD_REQUEST_TEXT = "请求格式有误：base 应为作为对比基准的运行 ID" +BASE_NOT_FOUND = "作为对比基准的运行记录不存在" +SAME_RUN = "不能和同一次运行自己对比，请选择另一次运行作为基准" +UNSAVED = "{who}不属于任何已保存的工作流，无法确认两次运行是同一个工作流，不能对比" +OTHER_WORKFLOW = "两次运行不属于同一个工作流（「{target}」与「{base}」），不能对比" +UNSEALED_TEXT = "{who}尚未结束并封存：对比只用封存的运行记录，运行结束后才能对比" +SEAL_BROKEN_TEXT = "{who}的封存核对不一致：封存后有事件被修改、删除或插入，不能用于对比。可在记录页点「核对」查看核对结果" +NO_BASE_TEXT = ("没有找到可以对比的上一次运行：需要同一工作流、同一运行类别（定时任务发起的优先找同一定时任务）、" + "更早结束并已封存的运行") +TARGET = "这次运行" +BASE = "作为基准的运行" + + +@router.get("/{run_id}/compare/base") +async def compare_base(run_id: str) -> dict[str, Any]: + """默认拿来对比的那一次运行：{base_run_id, reason: same_schedule | same_workflow, base}。""" + found = await default_base(run_id) + if found is None: + raise CodedHTTPException(404, NO_BASE_TEXT, NO_BASE) + run, reason = found + return {"base_run_id": run.id, "reason": reason, "base": run_ref(run)} + + +@router.get("/{run_id}/compare") +async def compare_with(run_id: str, base: str | None = None) -> dict[str, Any]: + """这次运行和 base 那一次的对比，形状见 engine/compare.py。""" + if base is None or not base.strip(): + raise CodedHTTPException(422, BAD_REQUEST_TEXT, BAD_REQUEST) + return await compare_runs(run_id, base.strip()) + + +async def default_base(run_id: str) -> tuple[Run, str] | None: + """(默认的基准运行, 怎么选的)；找不到返回 None，这次运行本身不存在 404。 + + 「早」按结束时间比：这次运行还没结束（还在跑、停在审批）的按开始时间。只挑终态、封存过的运行——没封存的拿来 + 对比只会得到 409。定时任务发起的先找同一定时任务（同一个工作流）的；它是这个任务的第一次时，退回同一工作流、 + 同一运行类别。没挂在已保存的工作流上的运行（画布上没保存就运行的图）没有可比的。""" + async with SessionLocal() as session: + run = await session.get(Run, run_id) + if run is None: + raise CodedHTTPException(404, "运行记录不存在", RUN_NOT_FOUND) + if not run.workflow_id: + return None + before = run.finished_at or run.started_at or run.created_at + query = (select(Run) + .where(Run.id != run.id, Run.workflow_id == run.workflow_id, Run.status.in_(_TERMINAL), + Run.manifest_hash.is_not(None), Run.finished_at.is_not(None), Run.finished_at < before) + .order_by(Run.finished_at.desc(), Run.created_at.desc()).limit(1)) + if run.schedule_id: + found = (await session.execute(query.where(Run.schedule_id == run.schedule_id))).scalars().first() + if found is not None: + return found, "same_schedule" + found = (await session.execute(query.where(Run.run_class == run.run_class))).scalars().first() + return (found, "same_workflow") if found is not None else None + + +async def compare_runs(run_id: str, base_id: str) -> dict[str, Any]: + """对比两次运行（run_id 是这次，base_id 是基准）。拒绝时抛 CodedHTTPException：运行不存在 404，同一次运行、 + 不是同一个工作流 422，没结束封存、封存核对不一致 409。""" + async with SessionLocal() as session: + target = await session.get(Run, run_id) + base = await session.get(Run, base_id) + kinds = {str(name): kind for name, kind in (await session.execute(select(DataSource.name, DataSource.kind)))} + if target is None: + raise CodedHTTPException(404, "运行记录不存在", RUN_NOT_FOUND) + if base is None: + raise CodedHTTPException(404, BASE_NOT_FOUND, RUN_NOT_FOUND) + if base.id == target.id: + raise CodedHTTPException(422, SAME_RUN, MISMATCH) + for run, who in ((target, TARGET), (base, BASE)): + if not run.workflow_id: + raise CodedHTTPException(422, UNSAVED.format(who=who), MISMATCH) + if target.workflow_id != base.workflow_id: + raise CodedHTTPException(422, OTHER_WORKFLOW.format(target=target.workflow_name, base=base.workflow_name), + MISMATCH) + sealed = [] + for run, who in ((target, TARGET), (base, BASE)): + found = await _sealed(run.id) + if not found.sealed or found.status not in _TERMINAL: + raise CodedHTTPException(409, UNSEALED_TEXT.format(who=who), UNSEALED) + if not found.trusted: + raise CodedHTTPException(409, SEAL_BROKEN_TEXT.format(who=who), SEAL_BROKEN) + sealed.append(found) + masks = _Masks() + target_side = await _side(target, sealed[0], masks, kinds) + base_side = await _side(base, sealed[1], masks, kinds) + return await asyncio.to_thread(engine.compare, base_side, target_side) + + +def run_ref(run: Run) -> dict[str, Any]: + """一次运行的概要（RunRef）：对比结果和默认基准里都用这一个形状。时间带时区（UTC）。""" + return {"id": run.id, "workflow_name": run.workflow_name, "version": run.version, "run_class": run.run_class, + "started_at": run.started_at.isoformat() if run.started_at else None, + "finished_at": run.finished_at.isoformat() if run.finished_at else None, + "schedule_id": run.schedule_id} + + +async def _side(run: Run, sealed: _Sealed, masks: _Masks, kinds: dict[str, Any]) -> engine.Side: + """一次运行交给对比的东西（engine.compare.Side）：封存范围内的事件、复验过哈希的口径卡、只认封存范围内工件的 + 读法、每份查询快照要遮的列，以及按口径卡那次执行还原输入用的 Sources。""" + cards = await asyncio.to_thread(_sealed_cards, sealed) + loader = _loader(sealed, cards) + # 遮罩要查库（数据源现在设的），在这里每份快照算一遍交进去，对比本身在线程里不碰库。快照先在线程里读好， + # 同一个 loader 记着读过的，对比时不再读盘 + snaps = await asyncio.to_thread(_read_all, loader, list(sealed.queries)) + masked = {artifact: frozenset(await _mask_of(snaps.get(artifact), info, masks)) + for artifact, info in sealed.queries.items()} + returned = _returned(sealed) + + def sources(started: int) -> recompute.Sources: + return recompute.Sources(load=loader, queries=tuple(sealed.queries), outputs=_outputs_before(sealed, started), + masked=masked, returned=returned, finished=_finished_before(sealed, started)) + + return engine.Side(run=run_ref(run), graph=sealed.graph, events=sealed.events, cards=cards, load=loader, + masked=masked, sources=sources, dialects=kinds, inputs=dict(run.input or {})) + + +def _read_all(loader: Callable[[str], Any], artifacts: list[str]) -> dict[str, Any]: + """读出这几份查询快照：读不出来、被改过的记 None（遮罩照样按台账记的数据源算）。""" + out: dict[str, Any] = {} + for artifact in artifacts: + try: + content = loader(artifact) + except ValueError: + content = None + out[artifact] = content if isinstance(content, dict) else None + return out diff --git a/backend/app/engine/compare.py b/backend/app/engine/compare.py new file mode 100644 index 00000000..32e022f0 --- /dev/null +++ b/backend/app/engine/compare.py @@ -0,0 +1,1152 @@ +"""运行对比：同一个工作流的两次运行（本周和上周的周报），把每个指标的变化拆清楚——是数据变了、口径变了， +还是模板 / SQL 变了；数据变化再按维度归因。每一项都追得到两边的查询快照。 + +只读：这里只算，不读库、不写工件、不追加事件。接口（api/compare.py）把两次运行封存范围内的事件、工件的读法和 +遮罩准备好交进来（Side），结果原样交回给发请求的人。 + +怎么对： + +- 查询：两次运行各自封存范围内交回过的查询快照，按「节点 + 该节点里第几次查询 + 工具」配对，配不上的再按规范化 + SQL（data/sql_semantics.normalize_sql）配。结构：规范文本和字面量都相同是 same；规范文本相同、只有字面量不同是 + params_only（定时运行每次的日期不同，属于参数变化不是模板变化）；否则 changed（数据源换了也算）。核查类的查询 + （双盲复算、反方审稿发的 role: verify）不进报告，不参与对比。 +- 口径卡：同一个口径卡节点按指标 id 配对；循环里执行了几次的取最后一次。表达式、口径版本、小数位、单位、显示格式 + 有一样不同就是口径变了：小数位改变取整，单位、比率与否改变数值的含义，都不只是换个写法。 +- 每个指标一个分类，按这个次序判：一边有一边没有（added / removed）；口径变了（caliber_and_data，可能连同数据）； + 喂它的查询结构变了、上游节点改了配置（sql_changed，标出、不拆）；输入对不上具体的查询，上游又有变化，说不清变化 + 从哪来（incomparable）；只有参数变了（params_only，含运行输入）；都没变时数值相等是 unchanged，不等是 data_only。 + 「喂它的查询」按输入的出处认：cell() 那一格、Agent 按出处交来的字段落在哪份快照上就是哪条查询（两边各自的快照 + 直接比 SQL）；不落在快照上的，看写它的节点自己查过的库，节点不查库（代码、数据整形……）就看它的配置和画布上的 + 全部上游。 +- 严格拆分（只对口径变了的）：旧表达式的每个叶子路径都能在这次的口径卡输入里取得值时， + 口径效应 = f_new(x_new) − f_old(x_new)，数据效应 = f_old(x_new) − f_old(x_old)。f_old(x_new) 用推演重算 + （engine/recompute.py）的还原和求值算，取整按旧口径卡的小数位；三个数按十进制精确相减，两效应之和逐位等于 + 总变化。取数的 SQL 也变了、用到了遮罩列、单位不同的不拆，说明为什么。 +- 维度归因：指标的输入落在查询快照的某一格（或 Agent 数组字段的一段行）、两边快照都没截断时，取快照里不是数值的 + 列作维度（遮罩的、日期类型的不算；两边都只有一个值的、两边没有一个值相同的——日期、期次这类每期都不同的列—— + 也不拿来配对），按维度键配对两边的行，算每个键的 base / target / delta，按 |delta| 取前 10。只有度量列之和 + 等于这个输入的值（两边都等）时才给占比；取的是合计行那一格时，其余各行之和等于它也算，合计行不列进去。 +- 遮罩列的值绝不出现在结果里：取自遮罩列的输入不给值，遮罩列不当维度、不当度量，用到遮罩列的指标不拆分（拆出来的 + f_old(x_new) 配上公开的数就能反推出遮住的那一格）。 +""" +from __future__ import annotations + +import ast +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field +from decimal import Decimal, InvalidOperation +from typing import Any + +from app.core.events import EventType +from app.data.sql_semantics import NormalizedSql, diff_facets, facets, normalize_sql +from app.engine import recompute +from app.engine.evidence import DEFAULT_FORMAT, RenderError, is_number, render_number +from app.engine.expressions import ExpressionError, cell_value, leaf_refs, parse_expression +from app.engine.labels import field_label +from app.engine.merge_query import MERGE_SOURCE +from app.engine.nodes.metrics import _REF_HEAD + +UNCHANGED = "unchanged" +DATA_ONLY = "data_only" +CALIBER_AND_DATA = "caliber_and_data" +SQL_CHANGED = "sql_changed" +PARAMS_ONLY = "params_only" +ADDED = "added" +REMOVED = "removed" +INCOMPARABLE = "incomparable" +#: summary 里的各类，次序同接口约定 +CLASSES = (UNCHANGED, DATA_ONLY, CALIBER_AND_DATA, SQL_CHANGED, PARAMS_ONLY, ADDED, REMOVED, INCOMPARABLE) +#: 这几类才做维度归因：数据（连同参数、口径）变了，两边的行说的是同一回事 +_ATTRIBUTED = frozenset({DATA_ONLY, PARAMS_ONLY, CALIBER_AND_DATA}) + +#: 维度归因最多列几个维度值（按变化的绝对值从大到小） +TOP_KEYS = 10 +#: 一条查询的结构差异最多写几条（diff_facets 已按对结果的影响从大到小排） +MAX_DIFFERENCES = 5 +#: 表达式、取值写进说明时最多几个字 +_SHOWN_MAX = 60 +#: 比较 SQL 时两边的叫法（diff_facets 的 labels） +_LABELS = ("基准", "这次") +#: 合并查询的 SQL 在 SQLite 里执行（engine/merge_query.py） +_MERGE_DIALECT = "sqlite" +#: 日期类的列不当维度：每期都不同，按它配对两边的行一个都配不上 +_TEMPORAL = frozenset({"date", "datetime", "time"}) +_FORMAT_LABEL = {"thousands": "千分位", "plain": "不分组", "percent_of_ratio": "比率按百分数显示"} + +NO_CARDS = "两次运行都没有口径卡，没有可以对比的指标" +EDGES_CHANGED = "节点之间的连线有变化" +DEFAULTS_CHANGED = "工作流默认设置有变化" +MASKED_NOTE = "部分列在数据源中设置了遮罩，对比中不显示这些列的值，也不用它们做拆分和归因" +NO_LINEAGE = "口径卡没有记录输入（生成于系统升级前），无法判断变化来源" +DATA_FROM_QUERIES = "取数的 SQL 没有变化，查询结果有变化" +DATA_FROM_UPSTREAM = "模板和取数的 SQL 都没有变化，变化来自上游数据" +SQL_NOT_SPLIT = "取数的 SQL 或上游配置有变化，变化无法在口径与数据之间拆分" +NOT_NUMBERS = "有一边的值不是数值，无法拆分" +SPLIT_MASKED = "用到了设置了遮罩的列，不做拆分" +NO_DIFFERENCE = "查询的写法有变化（选取的列、计算方式等）" +TRUNCATED = "查询结果被截断（{side}只取回了前 {rows} 行），不做维度归因" +MASKED_MEASURE = "度量列「{column}」设置了遮罩，不做维度归因" +UNREADABLE_SNAPSHOT = "查询快照无法读取，不做维度归因" +NOT_A_MEASURE = "度量列「{column}」有不是数值的格，不做维度归因" +NO_SHARE = "各维度值的度量之和与输入的值不相等，变化仅供参考，不给占比" +ZERO_TOTAL = "总变化为 0，不计算占比" +TOTAL_ROW = "输入取自合计行（第 {row} 行），按其余各行归因" +MORE_KEYS = "共 {count} 个维度值，列出变化最大的 {top} 个" +NO_OVERLAP = "列「{columns}」两边没有相同的值（可能是日期、期次这类每期都不同的列），不按它配对" + + +@dataclass +class Side: + """一次运行在对比里要用到的东西，接口从这次运行的封存范围里准备好。 + + - run：运行的概要（id、工作流名、版本、类别、起止时间、定时任务），原样放进结果 + - graph：运行时的图快照；events：封存范围内的事件 (seq, type, node_id, data) + - cards：封存范围内复验过哈希的口径卡 (节点, 工件, 指标集)（api/evidence._sealed_cards）。事件里记着、这里 + 没有的，就是读不出来或者被改过 + - load：工件 id → 内容，只认封存范围内的；文件不在返回 None，内容和哈希对不上抛 ValueError + - masked：查询快照 → 要遮的列（小写）：查询当时记下的加数据源现在设的 + - sources：口径卡那次执行开始的事件序号 → 推演重算还原输入要用的 Sources（节点产出、跑完的时刻都按那一刻）。 + 不给就按 load、masked 拼一个最小的：nodes.* 这类容器输入还原不出来 + - dialects：数据源名 → 类型（sqlite / postgres …），规范化 SQL 时按它的方言解析 + - inputs：运行输入（Run.input） + """ + + run: dict[str, Any] + graph: dict[str, Any] + events: list[tuple[int, str, str | None, Any]] + cards: list[tuple[str, str, dict[str, Any]]] = field(default_factory=list) + load: Callable[[str], Any] = lambda _: None + masked: Mapping[str, frozenset[str]] = field(default_factory=dict) + sources: Callable[[int], recompute.Sources] | None = None + dialects: Mapping[str, str | None] = field(default_factory=dict) + inputs: dict[str, Any] = field(default_factory=dict) + + +def compare(base: Side, target: Side) -> dict[str, Any]: + """对比两次运行（base 是基准，target 是这次）：{target, base, summary, template, metrics, queries, notes}， + 形状见接口约定（api/compare.py）。只读，不碰两次运行的任何东西。""" + return _Compare(base, target).result() + + +# -------------------------------------------------------------------------- +# 查询：收集、配对、比结构 +# -------------------------------------------------------------------------- + + +@dataclass +class _Query: + """一次运行里交回过的一份查询快照（核查类的不算）。""" + + artifact: str + node_id: str + tool: str | None + #: 这个节点里第几次查询，从 1 数（循环里执行几次的接着数） + ordinal: int + call_id: str | None = None + #: 快照里的 SQL；快照读不出来时取 tool.start 的参数（合并查询取 merge.end 的） + sql: str | None = None + snap: dict[str, Any] | None = None + #: 快照读不出来的原因：missing / tampered + problem: str | None = None + source: str | None = None + + +def _snapshot(load: Callable[[str], Any], artifact: str) -> tuple[dict[str, Any] | None, str | None]: + try: + content = load(artifact) + except ValueError: + return None, "tampered" + if isinstance(content, dict) and isinstance(content.get("rows"), list): + return content, None + return None, "missing" + + +def _queries(side: Side) -> list[_Query]: + """封存范围内交回过的查询快照，按交回的先后。 + + 调用工具节点、Agent 的查询在 tool.end 里（query_artifact）；合并查询只记在 node.finished 的台账里。同一件快照 + 两处都有的按 tool.end 算一次。核查类的查询（role: verify）不进写作目录，这里也不收。""" + out: list[_Query] = [] + seen: set[tuple[str, str]] = set() + asked: dict[tuple[str, Any], str] = {} # (节点, call_id) → tool.start 里的 SQL + merged: dict[str, str] = {} # 合并节点 → merge.end 里的 SQL + count: dict[str, int] = {} + + def add(node_id: str, artifact: str, tool: Any, call_id: Any, sql: Any, source: Any) -> None: + count[node_id] = count.get(node_id, 0) + 1 + snap, problem = _snapshot(side.load, artifact) + own = snap.get("sql") if snap is not None and isinstance(snap.get("sql"), str) else None + out.append(_Query(artifact=artifact, node_id=node_id, tool=str(tool) if tool else None, + ordinal=count[node_id], call_id=str(call_id) if call_id else None, + sql=own or (sql if isinstance(sql, str) and sql.strip() else None), snap=snap, + problem=problem, source=(snap or {}).get("source") or source)) + + for _, etype, node_id, data in side.events: + if not node_id or not isinstance(data, dict) or data.get("role") == "verify": + continue + if etype == EventType.TOOL_START: + args = data.get("args") + if isinstance(args, dict) and isinstance(args.get("sql"), str): + asked[(node_id, data.get("call_id"))] = args["sql"] + elif etype == EventType.TOOL_END and data.get("query_artifact"): + artifact = str(data["query_artifact"]) + seen.add((node_id, artifact)) + add(node_id, artifact, data.get("tool"), data.get("call_id"), asked.get((node_id, data.get("call_id"))), + None) + elif etype == EventType.MERGE_END and isinstance(data.get("sql"), str): + merged[node_id] = data["sql"] + elif etype == EventType.NODE_FINISHED: + for entry in data.get("evidence") or []: + if not (isinstance(entry, dict) and entry.get("kind") == "query" and entry.get("artifact")): + continue + artifact = str(entry["artifact"]) + if (node_id, artifact) in seen: + # tool.end 已经收过：台账里多记着数据源，读不出快照时用得上 + for q in out: + if q.node_id == node_id and q.artifact == artifact and not q.source: + q.source = entry.get("source") + continue + seen.add((node_id, artifact)) + add(node_id, artifact, entry.get("tool"), entry.get("call_id"), merged.get(node_id), + entry.get("source")) + return out + + +def _pair(base: list[_Query], target: list[_Query], + text: Callable[[_Query], str | None]) -> list[tuple[_Query | None, _Query | None]]: + """先按「节点 + 第几次 + 工具」配，配不上的再按规范化 SQL 配（节点换了 ID、同一个模板只有日期不同的也配得上）。 + 结果按这次运行的先后排，基准运行多出来的接在后面。""" + slots = {(q.node_id, q.ordinal, q.tool): i for i, q in enumerate(base)} + matched: dict[int, int] = {} + for j, q in enumerate(target): + i = slots.get((q.node_id, q.ordinal, q.tool)) + if i is not None: + matched[j] = i + taken = set(matched.values()) + for j, q in enumerate(target): + if j in matched or (key := text(q)) is None: + continue + i = next((i for i, b in enumerate(base) if i not in taken and text(b) == key), None) + if i is not None: + matched[j] = i + taken.add(i) + pairs: list[tuple[_Query | None, _Query | None]] = [ + (base[matched[j]] if j in matched else None, q) for j, q in enumerate(target)] + return pairs + [(b, None) for i, b in enumerate(base) if i not in taken] + + +@dataclass +class _Shape: + """一对查询的结构：same / params_only / changed / added / removed，不同的字面量，结构上差在哪。""" + + structure: str + literals: list[dict[str, str]] = field(default_factory=list) + differences: list[str] = field(default_factory=list) + #: 两边的 SQL 都拿得到（拿不到的只能说不知道，不能说变了） + known: bool = True + + +# -------------------------------------------------------------------------- +# 口径卡 +# -------------------------------------------------------------------------- + + +@dataclass +class _Card: + """一个口径卡节点在一次运行里最后一次执行的指标集。""" + + node_id: str + artifact: str + content: dict[str, Any] | None + #: 指标集读不出来、被改过的说法 + problem: str | None + #: 那次执行的 node.started 序号：输入按那一刻的节点产出还原 + started: int + #: 执行了几次 + runs: int + plans: dict[str, Any] = field(default_factory=dict) + #: 这张卡上全部指标的输入，按路径（同一次执行里同一条路径的值是同一个） + inputs: dict[str, Any] = field(default_factory=dict) + sources: recompute.Sources | None = None + + def metrics(self) -> list[dict[str, Any]]: + return [m for m in (self.content or {}).get("metrics") or [] if isinstance(m, dict) and m.get("id") is not None] + + +def _cards(side: Side) -> dict[str, _Card]: + """每个口径卡节点最后一次执行的指标集，按第一次执行的先后。哪些 node.finished 算口径卡的执行，和 + api/evidence._sealed_cards 一样认：台账里的 metric_set 条目；一期以前没有台账的，画布上是口径卡的节点的产出。""" + verified = {(node_id, artifact): card for node_id, artifact, card in side.cards} + calibers = {str(n.get("id")) for n in side.graph.get("nodes") or [] + if isinstance(n, dict) and n.get("type") == "metrics"} + runs: dict[str, list[tuple[int, str]]] = {} + started: dict[str, list[int]] = {} + for seq, etype, node_id, data in side.events: + if not node_id or not isinstance(data, dict): + continue + if etype == EventType.NODE_STARTED: + started.setdefault(node_id, []).append(seq) + if etype != EventType.NODE_FINISHED: + continue + arts = [str(e["artifact"]) for e in data.get("evidence") or [] + if isinstance(e, dict) and e.get("kind") == "metric_set" and e.get("artifact")] + if not arts and node_id in calibers and data.get("artifact"): + arts = [str(data["artifact"])] + runs.setdefault(node_id, []).extend((seq, art) for art in arts) + out: dict[str, _Card] = {} + for node_id, done in runs.items(): + if not done: + continue + seq, artifact = done[-1] + content = verified.get((node_id, artifact)) + out[node_id] = _Card(node_id=node_id, artifact=artifact, content=content, + problem=None if content is not None else _card_problem(side.load, artifact), + started=max((s for s in started.get(node_id, []) if s < seq), default=seq), + runs=len(done)) + return out + + +def _card_problem(load: Callable[[str], Any], artifact: str) -> str: + try: + content = load(artifact) + except ValueError: + return "指标集与哈希不一致，疑似被修改" + return "无法读取指标集" if content is None else "指标集的内容不完整" + + +def _plan_card(side: Side, card: _Card, queries: list[_Query]) -> None: + """按推演重算的规则还原卡上每个指标的输入：落在哪份快照上、是不是遮罩列、容器的完整值。""" + if card.content is None: + return + sources = side.sources(card.started) if side.sources is not None else recompute.Sources( + load=side.load, queries=tuple(dict.fromkeys(q.artifact for q in queries)), masked=side.masked) + store = recompute._Store(sources) + card.sources = sources + for metric in card.metrics(): + plan = recompute._plan(metric, store) + card.plans[str(metric["id"])] = plan + for path, inp in plan.inputs.items(): + card.inputs.setdefault(path, inp) + + +# -------------------------------------------------------------------------- +# 数：精确相减、渲染 +# -------------------------------------------------------------------------- + + +def _exact(value: Any) -> Decimal | None: + """有限的数换成十进制：浮点按最短的写法（repr），相减不带二进制的误差。""" + if not recompute._number(value): + return None + try: + return Decimal(repr(value)) if isinstance(value, float) else Decimal(value) + except (InvalidOperation, ValueError): + return None + + +def _plain(d: Decimal, *, whole: bool) -> int | float: + """十进制换回 JSON 里的数：参与的都是整数时给整数，否则给浮点。""" + if whole and d == d.to_integral_value(): + return int(d) + return float(d) + + +def _places(*metrics: dict[str, Any]) -> int | None: + """几个指标里最多的小数位；有一个没设的就不定小数位（按值本来的样子写）。""" + found = [m.get("decimals") for m in metrics] + if any(not isinstance(d, int) or isinstance(d, bool) for d in found): + return None + return max(found) + + +def _scale(metric: dict[str, Any]) -> tuple[str, bool]: + """数值的含义：单位，以及是不是比率（percent_of_ratio 的 0.087 和单位为 % 的 8.7 不能直接相减）。""" + return str(metric.get("unit") or ""), (metric.get("format") or DEFAULT_FORMAT) == "percent_of_ratio" + + +def _signed(meta: dict[str, Any], value: Any) -> str | None: + """变化量的写法：+300.0元、-2单、+10.3 个百分点（比率、百分数的变化写百分点，和推演同一个写法）。为 0 写不带 + 符号的 0。渲染不出来的给 None。""" + if not recompute._number(value): + return None + if value != 0: + return recompute._delta(meta, 0, value)[1] + fmt, unit, places = meta.get("format") or DEFAULT_FORMAT, str(meta.get("unit") or ""), meta.get("decimals") + try: + if fmt == "percent_of_ratio": + return render_number(0, decimals=places, fmt=fmt).rstrip("%") + " 个百分点" + if unit in ("%", "％"): + return render_number(0, decimals=places, fmt=fmt) + " 个百分点" + return render_number(0, unit=unit, decimals=places, fmt=fmt) + except RenderError: + return None + + +def _shown(value: Any) -> str: + """一个值写进说明：文本、数照写（太长的截断），空值写「空值」，容器写大概。""" + if value is None: + return "空值" + if isinstance(value, (list, dict)): + return f"[{len(value)} 项]" if isinstance(value, list) else f"{{{len(value)} 个字段}}" + text = str(value) + return text if len(text) <= _SHOWN_MAX else text[:_SHOWN_MAX] + "…" + + +def _expression(metric: dict[str, Any]) -> str: + """表达式的规范写法：空白、括号写法不同的算同一个。解析不了的照原文。""" + raw = str(metric.get("expression") or "").strip() + try: + return ast.unparse(parse_expression(raw)[0]) + except (ExpressionError, ValueError): + return raw + + +# -------------------------------------------------------------------------- +# 画布 +# -------------------------------------------------------------------------- + + +def _nodes(graph: dict[str, Any]) -> dict[str, dict[str, Any]]: + return {str(n.get("id")): n for n in graph.get("nodes") or [] if isinstance(n, dict) and n.get("id")} + + +def _data(node: dict[str, Any]) -> dict[str, Any]: + return node.get("data") if isinstance(node.get("data"), dict) else {} + + +def _config(node: dict[str, Any]) -> dict[str, Any]: + config = _data(node).get("config") + return config if isinstance(config, dict) else {} + + +def _edges(graph: dict[str, Any]) -> set[tuple[str, str, str]]: + """连线：(起点, 终点, 出口)。默认出口的几种写法算同一个（和 autofix 认连线的规则一致）。""" + out = set() + for edge in graph.get("edges") or []: + if isinstance(edge, dict): + handle = edge.get("sourceHandle") + out.add((str(edge.get("source")), str(edge.get("target")), + "" if handle in (None, "out", "source") else str(handle))) + return out + + +def _config_keys(was: dict[str, Any], now: dict[str, Any]) -> tuple[list[str], bool]: + """一个节点改了哪些配置项：(键名列表, 改的是不是只有名称)。值是字典的往下看一层（args.sql），不展示值本身。""" + keys: list[str] = [] + if was.get("type") != now.get("type"): + keys.append("type") + renamed = _data(was).get("label") != _data(now).get("label") + a, b = _config(was), _config(now) + for key in dict.fromkeys([*a, *b]): + x, y = a.get(key), b.get(key) + if x == y: + continue + if isinstance(x, dict) and isinstance(y, dict): + keys += [f"{key}.{sub}" for sub in dict.fromkeys([*x, *y]) if x.get(sub) != y.get(sub)] + else: + keys.append(key) + return (keys + ["label"] if renamed else keys), renamed and not keys + + +def _key_label(key: str, node_type: Any) -> str: + return "节点类型" if key == "type" else field_label(key, node_type) + + +# -------------------------------------------------------------------------- +# 输入的出处有没有变 +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _Change: + """一个输入的出处在两次运行之间变没变：same / params / sql / template / unknown，加一句说明。""" + + kind: str + reason: str | None = None + + +#: 合并几处出处变化时的轻重：模板、SQL 变了 > 说不清 > 只有参数变了 > 没变 +_WEIGHT = {"sql": 3, "template": 3, "unknown": 2, "params": 1, "same": 0} + + +def _worst(changes: Iterable[_Change]) -> _Change: + found = list(changes) + return max(found, key=lambda c: _WEIGHT[c.kind]) if found else _Change("same") + + +def _from_input(path: str, item: dict[str, Any]) -> bool: + head = _REF_HEAD.match(path) + return item.get("via") == "input" or (head is not None and head.group(1) == "input") + + +# -------------------------------------------------------------------------- +# 对比 +# -------------------------------------------------------------------------- + + +class _Compare: + def __init__(self, base: Side, target: Side) -> None: + self.base, self.target = base, target + self.notes: list[str] = [] + self.masked_seen = False + self._norm: dict[tuple[str, str | None], NormalizedSql] = {} + self._shapes: dict[tuple[int, int], _Shape] = {} + self._ancestors: dict[str, set[str]] = {} + self.base_queries, self.target_queries = _queries(base), _queries(target) + self.pairs = _pair(self.base_queries, self.target_queries, self._text) + self.template, self.changed = self._template() + self.titles = {**{k: self._title_of(n) for k, n in _nodes(base.graph).items()}, + **{k: self._title_of(n) for k, n in _nodes(target.graph).items()}} + self.parents: dict[str, set[str]] = {} + for source, target_id, _ in _edges(base.graph) | _edges(target.graph): + self.parents.setdefault(target_id, set()).add(source) + + # ---------------------------------------------------------------- 结果 + + def result(self) -> dict[str, Any]: + metrics = self._metrics() + queries = [self._query_view(b, t) for b, t in self.pairs] + self._run_notes() + if self.masked_seen: + self.notes.append(MASKED_NOTE) + summary = {name: 0 for name in CLASSES} + for m in metrics: + summary[m["classification"]] += 1 + return {"target": self.target.run, "base": self.base.run, "summary": summary, "template": self.template, + "metrics": metrics, "queries": queries, "notes": list(dict.fromkeys(self.notes))} + + def _title_of(self, node: dict[str, Any]) -> str: + return str(_data(node).get("label") or node.get("id")) + + def title(self, node_id: Any) -> str: + return self.titles.get(str(node_id), str(node_id)) + + def _run_notes(self) -> None: + """运行输入的变化：键名，值短的连值一起写。值可能是一大段文字，太长的只说有变化。""" + before, after = self.base.inputs or {}, self.target.inputs or {} + for key in dict.fromkeys([*before, *after]): + a, b = before.get(key), after.get(key) + if a == b: + continue + short = all(v is None or isinstance(v, (int, float, str, bool)) and len(str(v)) <= 40 for v in (a, b)) + self.notes.append(f"运行输入「{key}」由「{_shown(a)}」改为「{_shown(b)}」" if short + else f"运行输入「{key}」有变化") + + # ---------------------------------------------------------------- 查询 + + def _dialect(self, source: Any) -> str | None: + if source == MERGE_SOURCE: + return _MERGE_DIALECT + return self.target.dialects.get(str(source)) or self.base.dialects.get(str(source)) if source else None + + def _normalize(self, sql: str, dialect: str | None) -> NormalizedSql: + key = (sql, dialect) + if key not in self._norm: + self._norm[key] = normalize_sql(sql, dialect) + return self._norm[key] + + def _text(self, q: _Query) -> str | None: + return self._normalize(q.sql, self._dialect(q.source)).text if q.sql else None + + def shape(self, b: _Query | None, t: _Query | None) -> _Shape: + if b is None: + return _Shape("added") + if t is None: + return _Shape("removed") + key = (id(b), id(t)) + if key not in self._shapes: + self._shapes[key] = self._compare_sql(b, t) + return self._shapes[key] + + def _compare_sql(self, b: _Query, t: _Query) -> _Shape: + if b.sql is None or t.sql is None: + which = "基准运行" if b.sql is None else "这次运行" + return _Shape("changed", known=False, differences=[f"{which}的查询快照无法读取，无法比较 SQL"]) + differences = [] + if b.source and t.source and b.source != t.source: + differences.append(f"数据源由「{b.source}」改为「{t.source}」") + dialect = self._dialect(t.source or b.source) + nb, nt = self._normalize(b.sql, dialect), self._normalize(t.sql, dialect) + if nb.text != nt.text: + # 时间范围的不同排到最后：定时运行每期的日期本来就不同，结构上真正改了的（多了筛选、换了时间列……） + # 才是要先看的 + found = sorted(diff_facets(facets(b.sql, dialect), facets(t.sql, dialect), _LABELS), + key=lambda d: d.aspect == "time_range") + texts = [d.text for d in found] or [NO_DIFFERENCE] + return _Shape("changed", differences=(differences + texts)[:MAX_DIFFERENCES]) + if differences: + return _Shape("changed", differences=differences) + literals = [{"base": x, "target": y} for x, y in zip(nb.literals, nt.literals) if x != y] + return _Shape("params_only", literals) if literals else _Shape("same") + + def _query_view(self, b: _Query | None, t: _Query | None) -> dict[str, Any]: + shape = self.shape(b, t) + q = t or b + assert q is not None + view = { + "node_id": q.node_id, "node_title": self.title(q.node_id), "ordinal": q.ordinal, + "structure": shape.structure, + "base_sql": b.sql if b else None, "target_sql": t.sql if t else None, "literals": shape.literals, + "base_rows": len(b.snap["rows"]) if b and b.snap else None, + "target_rows": len(t.snap["rows"]) if t and t.snap else None, + "base_truncated": bool(b.snap.get("truncated")) if b and b.snap else None, + "target_truncated": bool(t.snap.get("truncated")) if t and t.snap else None, + "base_artifact": b.artifact if b else None, "target_artifact": t.artifact if t else None, + "differences": shape.differences, + } + problems = [f"{side}的查询快照{'与哈希不一致，疑似被修改' if one.problem == 'tampered' else '无法读取'}" + for side, one in (("基准运行", b), ("这次运行", t)) if one is not None and one.problem] + if problems: + view["problem"] = "；".join(problems) + return view + + def _side_query(self, queries: list[_Query], inp: Any) -> _Query | None: + """一个输入落在的那份快照，是这次运行里哪一次查询交回的（同一件快照出现几次时先找写它的节点）。""" + found = [q for q in queries if q.artifact == inp.snapshot] + node_id = inp.item.get("node_id") + return next((q for q in found if q.node_id == node_id), found[0] if found else None) + + def _query_change(self, b: _Query | None, t: _Query | None) -> _Change: + q = t or b + assert q is not None + name = f"查询「{self.title(q.node_id)}」" + if b is None: + return _Change("sql", f"节点「{self.title(q.node_id)}」这次多了一次查询") + if t is None: + return _Change("sql", f"节点「{self.title(q.node_id)}」这次少了一次查询") + shape = self.shape(b, t) + if not shape.known: + return _Change("unknown", f"{name}的快照无法读取，无法判断 SQL 是否有变化") + if shape.structure == "changed": + return _Change("sql", f"{name}的 SQL 结构有变化" + (f"：{shape.differences[0]}" if shape.differences else "")) + if shape.structure == "params_only": + shown = "，".join(f"{x['base']} → {x['target']}" for x in shape.literals[:2]) + more = f" 等 {len(shape.literals)} 处" if len(shape.literals) > 2 else "" + return _Change("params", f"{name}的 SQL 只有参数不同（{shown}{more}）") + return _Change("same") + + def _node_change(self, node_id: str) -> _Change | None: + """一个节点自己查过的库在两次运行之间变没变；它两边都没查过库返回 None。""" + own = [(b, t) for b, t in self.pairs if (b and b.node_id == node_id) or (t and t.node_id == node_id)] + return _worst(self._query_change(b, t) for b, t in own) if own else None + + def _upstream(self, node_id: str) -> set[str]: + """画布上 node_id 的全部上游（两次运行的图合起来，宁可多算）。""" + if node_id not in self._ancestors: + seen: set[str] = set() + todo = [node_id] + while todo: + for parent in self.parents.get(todo.pop(), ()): + if parent not in seen: + seen.add(parent) + todo.append(parent) + self._ancestors[node_id] = seen + return self._ancestors[node_id] + + def provenance(self, path: str, b: Any, t: Any) -> _Change: + """一个输入的出处在两次运行之间变没变（b、t 是推演重算还原出来的输入）。""" + if b is None or t is None: + return _Change("unknown", f"输入「{path}」只在一边的口径卡里有记录") + if b.snapshot or t.snapshot: + # 落在查询快照上：两边各自的那份快照直接比 SQL，不管全局配对把它们配给了谁 + if not (b.snapshot and t.snapshot): + return _Change("unknown", f"输入「{path}」只有一边对得上查询快照，无法比较取数") + qb, qt = self._side_query(self.base_queries, b), self._side_query(self.target_queries, t) + if qb is None or qt is None: + return _Change("unknown", f"输入「{path}」落在的查询快照不在封存范围内的查询里，无法比较取数") + return self._query_change(qb, qt) + nb, nt = b.item.get("node_id"), t.item.get("node_id") + if nb != nt: + return _Change("template", f"输入「{path}」的来源节点由「{self.title(nb)}」换成了「{self.title(nt)}」") + if _from_input(path, b.item): + if recompute._same(b.item.get("value"), t.item.get("value")): + return _Change("same") + return _Change("params", f"运行输入「{path}」由「{_shown(b.item.get('value'))}」" + f"改为「{_shown(t.item.get('value'))}」") + if nb and (own := self._node_change(str(nb))) is not None: + # 写它的节点自己查过库（Agent 没按出处交、调用工具节点的结果整份用）：它查的那几条就是喂它的查询 + if self.changed.get(str(nb)) and own.kind in ("same", "params"): + return _Change("template", f"上游节点「{self.title(nb)}」的配置有变化") + return own + if nb and self.changed.get(str(nb)): + return _Change("template", f"上游节点「{self.title(nb)}」的配置有变化") + # 写它的节点不查库（代码、数据整形、模型调用……）：看画布上它的全部上游；写它的节点不知道就看整张图 + scope = self._upstream(str(nb)) if nb else set(self.titles) + moved = [n for n in scope if self.changed.get(n)] + changes = [self._query_change(b2, t2) for b2, t2 in self.pairs if (t2 or b2).node_id in scope] + worst = _worst(changes) + where = f"来自「{self.title(nb)}」" if nb else "不知道由哪个节点写入" + if moved or worst.kind in ("sql", "template", "unknown"): + return _Change("unknown", f"输入「{path}」{where}，对不上具体的查询；它的上游有变化，无法判断变化来源") + return worst + + # ---------------------------------------------------------------- 模板 + + def _template(self) -> tuple[dict[str, Any], dict[str, str]]: + """两次运行的图快照比较：新增、删除的节点，配置有变化的节点和配置项名（不展示值）。changed 是影响取值的 + 变化（新增、删除、类型或配置变了；只改名称的不算），判断输入的出处时用。""" + before, after = _nodes(self.base.graph), _nodes(self.target.graph) + nodes: list[dict[str, Any]] = [] + changed: dict[str, str] = {} + + def entry(node_id: str, node: dict[str, Any], change: str, keys: list[str]) -> dict[str, Any]: + return {"node_id": node_id, "title": self._title_of(node), "change": change, "keys": keys, + "labels": [_key_label(k, node.get("type")) for k in keys]} + + for node_id, node in after.items(): + was = before.get(node_id) + if was is None: + nodes.append(entry(node_id, node, "added", [])) + changed[node_id] = "added" + continue + keys, renamed_only = _config_keys(was, node) + if keys: + nodes.append(entry(node_id, node, "config", keys)) + if not renamed_only: + changed[node_id] = "config" + for node_id, node in before.items(): + if node_id not in after: + nodes.append(entry(node_id, node, "removed", [])) + changed[node_id] = "removed" + same_edges = _edges(self.base.graph) == _edges(self.target.graph) + same_defaults = (self.base.graph.get("defaults") or {}) == (self.target.graph.get("defaults") or {}) + if not same_edges: + self.notes.append(EDGES_CHANGED) + if not same_defaults: + self.notes.append(DEFAULTS_CHANGED) + return {"base_version": self.base.run.get("version"), "target_version": self.target.run.get("version"), + "same_graph": not nodes and same_edges and same_defaults, "nodes": nodes}, changed + + # ---------------------------------------------------------------- 指标 + + def _metrics(self) -> list[dict[str, Any]]: + before, after = _cards(self.base), _cards(self.target) + if not before and not after: + self.notes.append(NO_CARDS) + return [] + out: list[dict[str, Any]] = [] + for node_id in [*after, *(n for n in before if n not in after)]: + b, t = before.get(node_id), after.get(node_id) + for side, card in (("基准运行", b), ("这次运行", t)): + if card is not None and card.runs > 1: + self.notes.append(f"口径卡「{self.title(node_id)}」在{side}中执行了 {card.runs} 次，按最后一次对比") + if b is not None: + _plan_card(self.base, b, self.base_queries) + if t is not None: + _plan_card(self.target, t, self.target_queries) + for side, card in (("基准运行", b), ("这次运行", t)): + if card is not None and card.problem: + self.notes.append(f"{side}的口径卡「{self.title(node_id)}」{card.problem}，其中的指标无法对比") + self._card_notes(node_id, b, t) + ids = [str(m["id"]) for m in (t.metrics() if t else [])] + ids += [mid for mid in (str(m["id"]) for m in (b.metrics() if b else [])) if mid not in ids] + for mid in ids: + bm = next((m for m in (b.metrics() if b else []) if str(m["id"]) == mid), None) + tm = next((m for m in (t.metrics() if t else []) if str(m["id"]) == mid), None) + out.append(self._metric(node_id, b, t, bm, tm)) + return out + + def _card_notes(self, node_id: str, b: _Card | None, t: _Card | None) -> None: + """口径卡本身的变化（口径名称、固定的来源版本）：不改变指标的分类，写进说明。""" + if b is None or t is None or b.content is None or t.content is None: + return + before, after = b.content.get("caliber") or "", t.content.get("caliber") or "" + if before != after: + self.notes.append(f"口径卡「{self.title(node_id)}」的口径名称由「{before}」改为「{after}」") + src_b, src_t = b.content.get("source"), t.content.get("source") + if src_b != src_t: + def version(src: Any) -> str: + return f"v{src.get('workflow_version')}" if isinstance(src, dict) else "本地定义" + self.notes.append(f"口径卡「{self.title(node_id)}」固定的口径来源由 {version(src_b)} 改为 {version(src_t)}") + + def _metric(self, card_id: str, b: _Card | None, t: _Card | None, bm: dict[str, Any] | None, + tm: dict[str, Any] | None) -> dict[str, Any]: + meta = tm or bm or {} + mid = str(meta.get("id")) + caliber = next((c.content.get("caliber") for c in (t, b) if c is not None and c.content), None) + out: dict[str, Any] = { + # 画布上的节点名；图里找不到这个节点（老运行的图读不出来）就写口径名 + "card_id": card_id, "card_title": self.title(card_id) if card_id in self.titles else caliber or card_id, + "metric_id": mid, "name": meta.get("name") or mid, "unit": meta.get("unit") or "", + "decimals": meta.get("decimals"), + "base": _value(bm), "target": _value(tm), + "base_rendered": _rendered(bm), "target_rendered": _rendered(tm), + "delta": None, "delta_rendered": None, "delta_pct": None, + "classification": INCOMPARABLE, "reasons": [], "decomposition": None, "decomposition_note": None, + "inputs": [], "attribution": None, + } + bp = b.plans.get(mid) if b is not None and bm is not None else None + tp = t.plans.get(mid) if t is not None and tm is not None else None + out["inputs"] = self._inputs(bp, tp) + # 一边有一边没有：整张卡都没有、卡里没有这个指标；卡读不出来的说不清有没有 + for card, metric, cls, side in ((b, bm, ADDED, "基准运行"), (t, tm, REMOVED, "这次运行")): + if metric is not None: + continue + if card is None: + out.update(classification=cls, reasons=[f"{side}没有口径卡「{self.title(card_id)}」"]) + elif card.content is None: + out["reasons"] = [f"{side}的口径卡「{self.title(card_id)}」{card.problem}，无法对比"] + else: + out.update(classification=cls, reasons=[f"{side}的口径卡「{self.title(card_id)}」里没有这个指标"]) + return out + assert bm is not None and tm is not None and b is not None and t is not None + caliber = _caliber_reasons(bm, tm, b.content or {}, t.content or {}) + self._delta(out, bm, tm) + lineage = isinstance(bm.get("inputs"), list) and isinstance(tm.get("inputs"), list) + if caliber: + out.update(classification=CALIBER_AND_DATA, reasons=caliber) + out["decomposition"], out["decomposition_note"] = self._decompose(bm, tm, b, t, bp, tp) + elif not lineage: + quiet = self.template["same_graph"] and all(self.shape(x, y).structure == "same" for x, y in self.pairs) + same = recompute._same(bm.get("value"), tm.get("value")) + out.update(classification=UNCHANGED if quiet and same else INCOMPARABLE, + reasons=[] if quiet and same else [NO_LINEAGE]) + else: + paths = list(dict.fromkeys([*tp.inputs, *bp.inputs])) + changes = [self.provenance(p, bp.inputs.get(p), tp.inputs.get(p)) for p in paths] + worst = _worst(changes) + reasons = list(dict.fromkeys(c.reason for c in changes if c.kind == worst.kind and c.reason)) + if worst.kind in ("sql", "template"): + out.update(classification=SQL_CHANGED, reasons=reasons, decomposition_note=SQL_NOT_SPLIT) + elif worst.kind == "unknown": + out.update(classification=INCOMPARABLE, reasons=reasons) + elif worst.kind == "params": + out.update(classification=PARAMS_ONLY, reasons=reasons) + elif recompute._same(bm.get("value"), tm.get("value")): + out.update(classification=UNCHANGED) + else: + linked = all(inp.snapshot for inp in tp.inputs.values()) and bool(tp.inputs) + out.update(classification=DATA_ONLY, reasons=[DATA_FROM_QUERIES if linked else DATA_FROM_UPSTREAM]) + for side, metric in (("基准运行", bm), ("这次运行", tm)): + if metric.get("status") == "missing_input": + out["reasons"].append(f"{side}这个指标缺少输入，没有值") + if out["classification"] in _ATTRIBUTED and bp is not None and tp is not None: + out["attribution"] = self._attribution(bp, tp) + if any(inp.masked for plan in (bp, tp) if plan is not None for inp in plan.inputs.values()): + self.masked_seen = True + return out + + def _delta(self, out: dict[str, Any], bm: dict[str, Any], tm: dict[str, Any]) -> None: + """总变化：两边都是数、数值的含义相同（单位一样、比率与否一样）才相减，按十进制精确算。""" + b, t = _exact(bm.get("value")), _exact(tm.get("value")) + if b is None or t is None or _scale(bm) != _scale(tm): + return + whole = all(isinstance(m.get("value"), int) for m in (bm, tm)) + delta = _plain(t - b, whole=whole) + meta = {**tm, "decimals": _places(bm, tm)} + out.update(delta=delta, delta_rendered=_signed(meta, delta) if delta != 0 else None, + delta_pct=float(round((t - b) / abs(b) * 100, 2)) if b != 0 else None) + + def _decompose(self, bm: dict[str, Any], tm: dict[str, Any], b: _Card, t: _Card, bp: Any, + tp: Any) -> tuple[dict[str, Any] | None, str | None]: + """严格拆分：(拆分, None) 或 (None, 为什么没法拆)。""" + old, new = _exact(bm.get("value")), _exact(tm.get("value")) + if old is None or new is None: + return None, NOT_NUMBERS + if _scale(bm) != _scale(tm): + return None, "单位或比率的写法不同，两边的数值不能直接相减，不做拆分" + try: + leaves = leaf_refs(parse_expression(str(bm.get("expression") or ""))[0]) + except ExpressionError as e: + return None, f"旧口径的表达式无法解析（{e}），无法严格拆分" + if bp is None or not isinstance(bm.get("inputs"), list): + return None, "基准运行的口径卡没有记录输入（生成于系统升级前），无法确认取数是否有变化，无法严格拆分" + missing = [p for p in leaves if p not in t.inputs] + if missing: + more = f"等 {len(missing)} 个输入" if len(missing) > 1 else "" + return None, f"旧口径用到的「{missing[0]}」{more}在这次运行的口径卡里没有记录，无法严格拆分" + used = [t.inputs[p] for p in leaves] + plans = [p for p in (bp, tp) if p is not None] + if any(inp.masked for inp in used) or any(inp.masked for plan in plans for inp in plan.inputs.values()): + return None, SPLIT_MASKED + for path in leaves: + change = self.provenance(path, bp.inputs.get(path), t.inputs[path]) + if change.kind in ("sql", "template", "unknown"): + return None, f"旧口径用到的输入取数也有变化，无法严格拆分：{change.reason}" + # f_old(x_new)：旧表达式配这次的输入，按推演重算的规则还原、求值，取整按旧口径卡的小数位 + synthetic = {"id": bm.get("id"), "expression": bm.get("expression"), "decimals": bm.get("decimals"), + "inputs": [inp.item for inp in used], "substituted": None, "value": None} + assert t.sources is not None + plan = recompute._plan(synthetic, recompute._Store(t.sources)) + if plan.problem: + return None, f"无法严格拆分：{plan.problem}" + for path in plan.tops: + if problem := plan.inputs[path].problem: + return None, f"无法严格拆分：这次运行的输入「{path}」无法还原（{problem}）" + value, reason = recompute._evaluate(plan, {p: inp.value for p, inp in plan.inputs.items()}) + if reason is not None: + return None, f"按旧口径在这次的数据上无法计算（{reason}），无法严格拆分" + middle = _exact(value) + if middle is None: + return None, "按旧口径在这次的数据上算出的不是有限的数值，无法严格拆分" + whole = all(isinstance(v, int) and not isinstance(v, bool) for v in (bm.get("value"), tm.get("value"), value)) + caliber, data = _plain(new - middle, whole=whole), _plain(middle - old, whole=whole) + meta = {**tm, "decimals": _places(bm, tm)} + return {"caliber_effect": caliber, "data_effect": data, "caliber_rendered": _signed(meta, caliber), + "data_rendered": _signed(meta, data)}, None + + def _inputs(self, bp: Any, tp: Any) -> list[dict[str, Any]]: + """两边的输入按路径对齐：值（遮罩列、核对不了遮没遮的不给），各自落在的查询快照。""" + b_in = bp.inputs if bp is not None else {} + t_in = tp.inputs if tp is not None else {} + out = [] + for path in dict.fromkeys([*t_in, *b_in]): + b, t = b_in.get(path), t_in.get(path) + entry: dict[str, Any] = {"path": path, "base": _input_value(b), "target": _input_value(t)} + if any(inp is not None and inp.masked for inp in (b, t)): + entry["masked"] = True + if b is not None and b.snapshot: + entry["base_artifact"] = b.snapshot + if t is not None and t.snapshot: + entry["target_artifact"] = t.snapshot + out.append(entry) + return out + + # ---------------------------------------------------------------- 维度归因 + + def _attribution(self, bp: Any, tp: Any) -> dict[str, Any] | None: + """挑一个落在查询快照上、两边都变了的输入做维度归因：先找给得出占比的，没有就取第一个列得出行的，再没有 + 就取第一个说明（截断、度量列遮罩……）。一个都挑不出来返回 None。""" + found: list[dict[str, Any]] = [] + for path, t in tp.inputs.items(): + b = bp.inputs.get(path) + if b is None or not (b.snapshot and t.snapshot) or recompute._same(b.value, t.value): + continue + if self.provenance(path, b, t).kind not in ("same", "params"): + continue + result = self._attribute(b, t) + if result is None: + continue + if result["rows"] and all(r["share"] is not None for r in result["rows"]): + return result + found.append(result) + return next((r for r in found if r["rows"]), found[0] if found else None) + + def _attribute(self, b: Any, t: Any) -> dict[str, Any] | None: + """一个输入的维度归因：{columns, measure, rows, total_delta, note}；归不了因的照实写在 note 里（rows 为空）。 + 快照里没有可以当维度的列（只有一行汇总这种）返回 None：没什么可归的。""" + measure = _locator(b).get("column") + if not isinstance(measure, str) or _locator(t).get("column") != measure: + return None + sb, st = self._snap(self.base_queries, b), self._snap(self.target_queries, t) + empty = {"columns": [], "measure": measure, "rows": [], "total_delta": None} + if sb is None or st is None: + return {**empty, "note": UNREADABLE_SNAPSHOT} + for side, snap in (("基准运行", sb), ("这次运行", st)): + if snap.get("truncated"): + # 截断的结果只是前几行：各键的和不是完整的和，按它归因会把没取回的行的变化算到别的键头上 + return {**empty, "note": TRUNCATED.format(side=side, rows=len(snap["rows"]))} + hidden = (self.base.masked.get(b.snapshot) or frozenset()) | (self.target.masked.get(t.snapshot) or frozenset()) + if any(str(c).lower() in hidden for c in [*(sb.get("columns") or []), *(st.get("columns") or [])]): + self.masked_seen = True + if measure.lower() in hidden: + return {**empty, "note": MASKED_MEASURE.format(column=measure)} + rb, rt = _records(sb, b), _records(st, t) + if rb is None or rt is None: + return None + values = (_input_total(b), _input_total(t)) + notes: list[str] = [] + if b.row is not None and t.row is not None and not _adds_up(rb, rt, measure, values): + # 取的是合计行那一格：其余各行之和等于它（两边都等）时按其余各行归因，合计行不列进去 + kb, kt = [x for x in rb if x[0] != b.row], [x for x in rt if x[0] != t.row] + if kb and kt and len(kb) < len(rb) and len(kt) < len(rt) and _adds_up(kb, kt, measure, values): + rb, rt = kb, kt + notes.append(TOTAL_ROW.format(row=t.row + 1)) + dims, apart = _dimensions(sb, st, rb, rt, measure, hidden) + if not dims: + return {**empty, "note": "；".join(apart)} if apart else None + notes += apart + groups_b, groups_t = _group(rb, dims, measure), _group(rt, dims, measure) + if groups_b is None or groups_t is None: + return {**empty, "note": NOT_A_MEASURE.format(column=measure)} + total_b, total_t = sum(groups_b.values(), Decimal(0)), sum(groups_t.values(), Decimal(0)) + total = total_t - total_b + # 占比只在这份结果正好是这个输入的拆分时才有意义：各键之和等于输入的值，两边都等 + share = _equal(values[0], total_b) and _equal(values[1], total_t) + if not share: + notes.append(NO_SHARE) + elif total == 0: + notes.append(ZERO_TOTAL) + rows = [] + for key in dict.fromkeys([*groups_b, *groups_t]): + vb, vt = groups_b.get(key), groups_t.get(key) + delta = (vt or Decimal(0)) - (vb or Decimal(0)) + rows.append((abs(delta), {"key": list(key), "base": None if vb is None else _plain(vb, whole=False), + "target": None if vt is None else _plain(vt, whole=False), + "delta": _plain(delta, whole=False), + "share": float(delta / total) if share and total != 0 else None})) + rows.sort(key=lambda r: -r[0]) # 稳定排序：变化一样大的按先出现的 + if len(rows) > TOP_KEYS: + notes.append(MORE_KEYS.format(count=len(rows), top=TOP_KEYS)) + return {"columns": dims, "measure": measure, "rows": [r for _, r in rows[:TOP_KEYS]], + "total_delta": _plain(total, whole=False), "note": "；".join(notes) or None} + + def _snap(self, queries: list[_Query], inp: Any) -> dict[str, Any] | None: + q = self._side_query(queries, inp) + return q.snap if q is not None else None + + +# -------------------------------------------------------------------------- +# 指标、输入的几样小事 +# -------------------------------------------------------------------------- + + +def _value(metric: dict[str, Any] | None) -> Any: + """指标的值：有限的数才给，别的（空值、文本、溢出成无穷大的）给 None，写法看 rendered。""" + value = (metric or {}).get("value") + return value if recompute._number(value) else None + + +def _rendered(metric: dict[str, Any] | None) -> str | None: + """卡上记的写法（报告里就是这几个字）；没记的按卡上的格式渲染。""" + if metric is None: + return None + rendered = metric.get("rendered") + return rendered if isinstance(rendered, str) else recompute._render(metric, metric.get("value")) + + +def _caliber_reasons(bm: dict[str, Any], tm: dict[str, Any], bc: dict[str, Any], tc: dict[str, Any]) -> list[str]: + """口径变了的地方，一处一句；没变返回空列表。""" + out = [] + if _expression(bm) != _expression(tm): + out.append(f"表达式由「{_shown(bm.get('expression'))}」改为「{_shown(tm.get('expression'))}」") + before, after = str(bc.get("caliber_version") or ""), str(tc.get("caliber_version") or "") + if before != after: + out.append(f"口径版本由 {before or '未设置'} 改为 {after or '未设置'}") + if bm.get("decimals") != tm.get("decimals"): + def places(d: Any) -> str: + return "未设置" if d is None else str(d) + out.append(f"小数位由 {places(bm.get('decimals'))} 改为 {places(tm.get('decimals'))}") + if (bm.get("unit") or "") != (tm.get("unit") or ""): + out.append(f"单位由「{bm.get('unit') or '无'}」改为「{tm.get('unit') or '无'}」") + fb, ft = bm.get("format") or DEFAULT_FORMAT, tm.get("format") or DEFAULT_FORMAT + if fb != ft: + out.append(f"显示格式由「{_FORMAT_LABEL.get(fb, fb)}」改为「{_FORMAT_LABEL.get(ft, ft)}」") + return out + + +def _input_value(inp: Any) -> Any: + """输入在对比里给人看的值：遮罩列、落在读不出来的快照上的不给；容器写卡上记的大概。""" + if inp is None or inp.masked or (inp.unreadable and inp.shape == "scalar"): + return None + if inp.shape == "container": + return inp.item.get("value") + return recompute._finite(inp.item.get("value")) + + +def _locator(inp: Any) -> dict[str, Any]: + locator = inp.item.get("locator") + return locator if isinstance(locator, dict) else {} + + +def _records(snap: dict[str, Any], inp: Any) -> list[tuple[int, dict[str, Any]]] | None: + """归因用到的行：(行号, {列: 值})。数组字段只用它那一段行，单格用整份结果。""" + columns = [str(c) for c in snap.get("columns") or []] + rows = snap.get("rows") or [] + first, last = (inp.rows if inp.rows is not None else (0, len(rows) - 1)) + out = [] + for i in range(first, min(last, len(rows) - 1) + 1): + row = rows[i] + if isinstance(row, dict): + out.append((i, {str(k): v for k, v in row.items()})) + elif isinstance(row, (list, tuple)) and len(row) == len(columns): + out.append((i, dict(zip(columns, row)))) + else: + return None + return out + + +def _kind(snap: dict[str, Any], column: str) -> str | None: + types = snap.get("column_types") + kind = types.get(column) if isinstance(types, dict) else None + return kind if isinstance(kind, str) else None + + +def _numeric(values: Iterable[Any], kind: str | None) -> bool: + """这一列是数值列：查询时记的类型是 number，或者没记类型、值（数字文本也算）全是数。""" + if kind is not None: + return kind == "number" + present = [v for v in values if v is not None] + return bool(present) and all(is_number(cell_value(v)) for v in present) + + +def _dimensions(sb: dict[str, Any], st: dict[str, Any], rb: list[tuple[int, dict[str, Any]]], + rt: list[tuple[int, dict[str, Any]]], measure: str, + hidden: frozenset[str] | set[str]) -> tuple[list[str] | None, list[str]]: + """维度列：(列名列表, 说明)。两边都有、不是数值、不是日期类型、没遮罩的列；两边各自都只有一个值的不算（分不出 + 维度）；两边没有一个值相同的不拿来配对（日期、期次这类每期都不同的列），写进说明。一个候选都没有返回 None。""" + common = [c for c in (str(c) for c in sb.get("columns") or []) if c in {str(x) for x in st.get("columns") or []}] + candidates = [] + for column in common: + if column == measure or column.lower() in hidden: + continue + kinds = (_kind(sb, column), _kind(st, column)) + if any(k in _TEMPORAL for k in kinds): + continue + vb, vt = [r.get(column) for _, r in rb], [r.get(column) for _, r in rt] + if _numeric(vb, kinds[0]) or _numeric(vt, kinds[1]): + continue + candidates.append((column, vb, vt)) + if not candidates: + return None, [] + dims, apart = [], [] + for column, vb, vt in candidates: + seen_b, seen_t = {_hashable(v) for v in vb}, {_hashable(v) for v in vt} + if len(seen_b) <= 1 and len(seen_t) <= 1: + continue + if seen_b and seen_t and not seen_b & seen_t: + apart.append(column) + continue + dims.append(column) + return dims, ([NO_OVERLAP.format(columns="、".join(apart))] if apart else []) + + +def _hashable(value: Any) -> Any: + return value if isinstance(value, (str, int, float, bool)) or value is None else repr(value) + + +def _measure(raw: Any) -> Decimal | None: + """度量列的一格，十进制：空值按 0（SQL 的 SUM 也不计空值），数字文本按数，别的不是数返回 None。""" + return Decimal(0) if raw is None else _exact(cell_value(raw)) + + +def _group(rows: list[tuple[int, dict[str, Any]]], dims: list[str], + measure: str) -> dict[tuple[Any, ...], Decimal] | None: + """按维度键汇总度量：{键: 和}（去掉了期次列之后同一个键可能有几行）。有不是数值的格返回 None。""" + out: dict[tuple[Any, ...], Decimal] = {} + for _, row in rows: + number = _measure(row.get(measure)) + if number is None: + return None + key = tuple(_hashable(row.get(d)) for d in dims) + out[key] = out.get(key, Decimal(0)) + number + return out + + +def _input_total(inp: Any) -> Decimal | None: + """输入的值（数组字段是各项之和），十进制。不是数的给 None：这时不给占比。""" + value = inp.value + if isinstance(value, list): + parts = [_exact(cell_value(v)) for v in value if v is not None] + return sum(parts, Decimal(0)) if all(p is not None for p in parts) else None # type: ignore[arg-type] + return _exact(cell_value(value)) + + +def _adds_up(rb: list[tuple[int, dict[str, Any]]], rt: list[tuple[int, dict[str, Any]]], measure: str, + values: tuple[Decimal | None, Decimal | None]) -> bool: + """度量列之和等于输入的值（两边都等）。""" + for rows, value in ((rb, values[0]), (rt, values[1])): + parts = [_measure(row.get(measure)) for _, row in rows] + if any(p is None for p in parts) or not _equal(value, sum(parts, Decimal(0))): # type: ignore[arg-type] + return False + return True + + +def _equal(value: Decimal | None, total: Decimal) -> bool: + """输入的值和度量之和相等：十进制逐位比；浮点算出来的值差在最后几位的也算（相对误差 1e-9 以内）。""" + if value is None: + return False + if value == total: + return True + scale = max(abs(value), abs(total)) + return scale != 0 and abs(value - total) <= scale * Decimal("1e-9") diff --git a/backend/tests/fixtures/compare_runs.py b/backend/tests/fixtures/compare_runs.py new file mode 100644 index 00000000..1d8bb6e2 --- /dev/null +++ b/backend/tests/fixtures/compare_runs.py @@ -0,0 +1,211 @@ +"""运行对比的虚构运行（tests/test_compare_engine.py、tests/test_compare_api.py 共用）。 + +两次周报运行照真实运行的事件形状手写（区域、金额、日期都是编的）：取数节点 fetch 查一条按区域汇总的 SQL，Agent 节点 +bot 自己再查一次、把各区域的金额作为数组字段 amounts 交给口径卡，口径卡算几个指标。工件按内容寻址放在字典里， +接口测试把它们原样落进工件库、事件落进运行记录。 +""" +from __future__ import annotations + +import json + +from app.core.artifact_store import canonical_json, content_hash +from app.engine import compare, recompute +from app.engine.evidence import render_number +from app.engine.expressions import parse_expression, substitute + +TOOL = "db_query__shop" +WEEK36 = ("2026-08-31", "2026-09-06") +WEEK37 = ("2026-09-07", "2026-09-13") +COLUMNS = ["region", "gmv", "refund"] +TYPES = {"region": "text", "gmv": "number", "refund": "number"} +BASE_ROWS = [["东区", 1200.5, 100.0], ["南区", 800.0, 50.0], ["西区", 600.0, 0.0]] +NEXT_ROWS = [["东区", 1500.5, 120.0], ["南区", 700.0, 40.0], ["西区", 600.0, 10.0]] +EAST = "cell(nodes.fetch, 0, 'gmv')" +REFUND = "cell(nodes.fetch, 0, 'refund')" + + +def region_sql(days=WEEK36, extra=""): + return ("SELECT region, SUM(amount) AS gmv, SUM(refund) AS refund FROM orders " + f"WHERE day BETWEEN '{days[0]}' AND '{days[1]}'{extra} GROUP BY region ORDER BY region") + + +def node(nid, ntype, label, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +def weekly_graph(sql=None, *, between=(), card_label="周报口径卡"): + return chain( + node("start", "input", "开始"), + node("fetch", "tool", "区域取数", tool=TOOL, args={"sql": sql or region_sql()}), + node("bot", "agent", "区域分析", prompt="按区域列出金额", tools=[TOOL], cite_fields=True), + *between, + node("card", "metrics", card_label, caliber="周报口径", caliber_version="v1"), + node("write", "report", "写周报"), + node("out", "output", "成果"), + ) + + +class Fake: + """一次虚构的运行：工件按内容寻址放在字典里，事件照真实运行的形状写（tool.start / tool.end / node.finished + 的证据台账、口径卡的 metric_set 条目、节点产出工件）。""" + + def __init__(self, run_id, *, graph=None, version=3, inputs=None, masked=None, source="shop"): + self.id = run_id + self.source = source + self.graph = graph or weekly_graph() + self.version = version + self.inputs = inputs or {} + self.masked = masked or {} + self.store: dict[str, object] = {} + self.tampered: set[str] = set() + self.events: list[tuple[int, str, str | None, dict]] = [] + self.cards: list[tuple[str, str, dict]] = [] + self.queries: list[str] = [] + self.elapsed = 0 + self.add("run.started", None, {"nodes": len(self.graph["nodes"])}) + + def put(self, obj): + text = canonical_json(obj) + artifact = content_hash(text) + self.store[artifact] = json.loads(text) + return artifact + + def load(self, artifact): + if artifact in self.tampered: + raise ValueError("工件内容与哈希不符，疑似被篡改") + return self.store.get(artifact) + + def add(self, etype, node_id, data): + self.events.append((len(self.events) + 1, etype, node_id, data)) + + def snapshot(self, sql, columns, rows, *, truncated=False, types=None, masks=None): + self.elapsed += 1 # 真实快照带着耗时：同一条 SQL 查两次也是两件工件 + snap = {"columns": columns, "rows": rows, "row_count": len(rows), "truncated": truncated, + "elapsed_ms": self.elapsed, "sql": sql, "source": self.source, + **({"column_types": types} if types else {}), **({"mask_columns": masks} if masks else {})} + artifact = self.put(snap) + self.queries.append(artifact) + return artifact + + def entry(self, node_id, artifact, columns, rows, *, tool=TOOL, call_id=None, truncated=False): + return {"kind": "query", "node_id": node_id, "exec": 1, "artifact": artifact, "tool": tool, + "source": self.source, + "columns": columns, "rows": len(rows), "truncated": truncated, + **({"call_id": call_id} if call_id else {})} + + def tool(self, node_id, sql, columns=COLUMNS, rows=BASE_ROWS, *, tool=TOOL, truncated=False, types=TYPES, + masks=None): + """调用工具节点查一次库。masks 是查询当时数据源设的遮罩（记进快照的 mask_columns）。""" + artifact = self.snapshot(sql, columns, rows, truncated=truncated, types=types, masks=masks) + output = self.put({"columns": columns, "rows": rows, "sql": sql, "source": self.source, "artifact": artifact}) + self.add("node.started", node_id, {"node_type": "tool"}) + self.add("tool.start", node_id, {"tool": tool, "args": {"sql": sql}}) + self.add("tool.end", node_id, {"tool": tool, "duration_ms": 3, "query_artifact": artifact}) + self.add("node.finished", node_id, {"duration_ms": 4, "attempt": 1, "artifact": output, "evidence": [ + self.entry(node_id, artifact, columns, rows, tool=tool, truncated=truncated)]}) + return artifact + + def agent(self, node_id, sql, columns=COLUMNS, rows=BASE_ROWS, *, types=TYPES, truncated=False, masks=None, + verify=()): + """Agent 节点自己查一次库，按出处交回数组字段 amounts（快照 gmv 列的全部行）。verify 是同一节点里核查类的 + 查询（role: verify，不进台账）。""" + artifact = self.snapshot(sql, columns, rows, truncated=truncated, types=types, masks=masks) + last = len(rows) - 1 + gmv = columns.index("gmv") + data = {"amounts": [r[gmv] for r in rows]} + evidence = {"amounts": {"ref": f"Q1.r0-{last}.gmv", "call": "Q1", "artifact": artifact, + "locator": {"rows": [0, last], "column": "gmv"}, "status": "verified"}} + output = self.put({"text": "按区域列出了金额", "data": data, "data_evidence": evidence}) + self.add("node.started", node_id, {"node_type": "agent"}) + self.add("tool.start", node_id, {"tool": TOOL, "args": {"sql": sql}, "call_id": "c1"}) + self.add("tool.end", node_id, {"tool": TOOL, "call_id": "c1", "duration_ms": 5, "query_artifact": artifact}) + for i, extra in enumerate(verify): + check = self.snapshot(extra, ["n"], [[1]]) + self.add("tool.end", node_id, {"tool": TOOL, "call_id": f"v{i}", "role": "verify", "query_artifact": check}) + self.add("node.finished", node_id, {"duration_ms": 9, "attempt": 1, "artifact": output, "evidence": [ + self.entry(node_id, artifact, columns, rows, call_id="c1", truncated=truncated)]}) + return artifact + + def code(self, node_id, value, var): + """沙箱代码节点算出一个数写进变量：它不查库，对不上具体的查询。""" + output = self.put({var: value}) + self.add("node.started", node_id, {"node_type": "code"}) + self.add("node.finished", node_id, {"duration_ms": 2, "attempt": 1, "artifact": output}) + + def card(self, metrics, *, node_id="card", caliber="周报口径", version="v1", runs=1): + card = {"kind": "metric_set", "caliber": caliber, "caliber_version": version, "metrics": metrics} + artifact = self.put(card) + for _ in range(runs): + self.add("node.started", node_id, {"node_type": "metrics"}) + self.add("node.finished", node_id, {"duration_ms": 1, "attempt": 1, "artifact": self.put({**card, "text": "…"}), + "evidence": [{"kind": "metric_set", "node_id": node_id, "exec": 1, + "artifact": artifact, "caliber": caliber, + "version": version}]}) + self.cards.append((node_id, artifact, card)) + return artifact + + def side(self): + def sources(started): + outputs = {} + for seq, etype, nid, data in self.events: + if seq < started and etype == "node.finished" and data.get("artifact"): + outputs[nid] = data["artifact"] + return recompute.Sources(load=self.load, queries=tuple(self.queries), outputs=outputs, masked=self.masked) + + run = {"id": self.id, "workflow_name": "区域周报", "version": self.version, "run_class": "formal", + "started_at": "2026-09-07T01:00:00+00:00", "finished_at": "2026-09-07T01:02:00+00:00", + "schedule_id": None} + return compare.Side(run=run, graph=self.graph, events=self.events, cards=self.cards, load=self.load, + masked=self.masked, sources=sources, dialects={self.source: "sqlite"}, inputs=self.inputs) + + +def cell_in(row, column, value, artifact, node_id="fetch"): + return {"path": f"cell(nodes.{node_id}, {row}, '{column}')", "value": value, "node_id": node_id, "via": "tool_cell", + "locator": {"row": row, "column": column}, "artifact": artifact, + "eid": f"ev:cell:{artifact[:8]}:{row}:{column}", "status": "verified"} + + +def amounts_in(artifact, rows): + last = len(rows) - 1 + return {"path": "vars.kpi.amounts", "value": f"[{len(rows)} 项]", "node_id": "bot", "via": "agent_field", + "field": "amounts", "ref": f"Q1.r0-{last}.gmv", "artifact": artifact, + "locator": {"rows": [0, last], "column": "gmv"}, "eid": f"ev:rows:{artifact[:8]}", "status": "verified"} + + +def scalar_in(path, value, node_id, via): + return {"path": path, "value": value, "node_id": node_id, "via": via, "field": path.split(".", 2)[-1], + "status": "ok" if value is not None else "missing"} + + +def metric(mid, expression, value, inputs, *, name=None, unit="元", decimals=1, fmt="thousands"): + tree = parse_expression(expression)[0] + scalars = {i["path"]: i["value"] for i in inputs if not str(i["value"]).endswith(" 项]")} + return {"id": mid, "name": name or mid, "unit": unit, "value": value, "expression": expression, + "decimals": decimals, "format": fmt, + "rendered": render_number(value, unit=unit, decimals=decimals, fmt=fmt), + "inputs": inputs, "substituted": substitute(tree, scalars), "recompute_ok": True, + "status": "ok" if value is not None else "missing_input"} + + +def week(run_id, rows=BASE_ROWS, *, sql=None, metrics=None, graph=None, version="v1", bot_rows=None, **kw): + """一次周报运行：fetch 和 bot 各查一次，口径卡算 east（东区金额）和 total（各区域合计）。 + metrics(fetch, bot, rows) 返回口径卡的指标，不给就是这两个。""" + run = Fake(run_id, graph=graph or weekly_graph(sql), **kw) + sql = sql or region_sql() + fetch = run.tool("fetch", sql, rows=rows) + bot_rows = bot_rows if bot_rows is not None else rows + bot = run.agent("bot", sql, rows=bot_rows) + if metrics is None: + metrics = default_metrics + run.card(metrics(fetch, bot, rows, bot_rows), version=version) + return run + + +def default_metrics(fetch, bot, rows, bot_rows): + total = sum(r[1] for r in bot_rows) + return [metric("east", EAST, rows[0][1], [cell_in(0, "gmv", rows[0][1], fetch)], name="东区金额"), + metric("total", "sum(vars.kpi.amounts)", total, [amounts_in(bot, bot_rows)], name="各区域合计")] diff --git a/backend/tests/test_compare_api.py b/backend/tests/test_compare_api.py new file mode 100644 index 00000000..7e92273a --- /dev/null +++ b/backend/tests/test_compare_api.py @@ -0,0 +1,387 @@ +"""运行对比的接口：GET /api/runs/{run_id}/compare/base 和 GET /api/runs/{run_id}/compare?base=。 + +两次周报运行用 tests/fixtures/compare_runs.py 的虚构运行：工件落进工件库，事件落进运行记录，按 runner 的算法封存 +（不连库、不调模型）。另有一组真跑：虚构的 SQLite 库在两次运行之间改了数，再一次代进去的日期参数不同。守的是： +- 两次都结束、封存且核对一致才比：没封存、还在运行、封存后事件被改过都回 409，说清是哪一次；不是同一个工作流、 + 同一次运行回 422；运行不存在 404 +- 对比只读：两次运行的 last_seq、事件、工件引用、工件文件一样不多 +- 遮罩列（查询当时记下的、数据源现在设的）的值不出现在结果里 +- 默认基准：同一定时任务优先，否则同一工作流、同一运行类别；只挑更早结束、封存过的终态运行 +- 真跑：同一条 SQL 数据变了是 data_only；只是代进去的日期不同是 params_only，图一样 +""" +from __future__ import annotations + +import asyncio +import json +import sqlite3 +from datetime import datetime, timedelta, timezone + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import func, select + +from app.core.artifact_store import manifest_hash, put_json +from app.core.config import settings +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import Artifact, DataSource, Run, RunEvent, Workflow +from app.engine.runner import run_manager, verify_manifest +from app.main import app +from tests.fixtures.compare_runs import ( + BASE_ROWS, + EAST, + NEXT_ROWS, + Fake, + amounts_in, + cell_in, + chain, + metric, + node, + week, +) +from tests.fixtures.sources import drop_source + +SOURCE = "cmp_shop" +T0 = datetime(2026, 9, 7, 1, 0, tzinfo=timezone.utc) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def new_workflow(name="区域周报") -> str: + async with SessionLocal() as session: + workflow = Workflow(name=name) + session.add(workflow) + await session.commit() + return workflow.id + + +async def persist(fake: Fake, workflow_id: str | None, *, finished=T0, status="succeeded", run_class="formal", + schedule_id=None, seal=True) -> str: + """把一次虚构运行落库：工件进工件库（内容寻址，id 和虚构时一样），事件进运行记录，按 runner 的算法封存。""" + for content in fake.store.values(): + await put_json(content, kind="compare_fixture") + async with SessionLocal() as session: + run = Run(workflow_id=workflow_id, workflow_name="区域周报", status=status, graph=fake.graph, + input=dict(fake.inputs), run_class=run_class, version=fake.version, schedule_id=schedule_id, + started_at=finished - timedelta(minutes=2), finished_at=finished if status != "running" else None, + agent_limits={}) + session.add(run) + await session.flush() + rows = [] + for seq, etype, node_id, data in fake.events: + session.add(RunEvent(run_id=run.id, seq=seq, type=etype, node_id=node_id, ts=float(seq), data=data)) + rows.append((seq, etype, node_id, data)) + run.last_seq = len(rows) + if seal: + run.manifest_hash, run.manifest_seq = manifest_hash(rows), rows[-1][0] + await session.commit() + return run.id + + +async def pair(base=None, target=None, **kw) -> tuple[str, str, str]: + """(工作流, 基准运行, 这次运行)：同一个工作流的两次周报，这次的数变了。""" + workflow = await new_workflow() + base_id = await persist(base or week("r1", source=SOURCE), workflow, finished=T0) + target_id = await persist(target or week("r2", NEXT_ROWS, source=SOURCE), workflow, + finished=T0 + timedelta(days=7), **kw) + return workflow, base_id, target_id + + +async def footprint(*run_ids: str) -> tuple: + """几次运行留在库里、盘上的痕迹：各自的 last_seq、封存、事件数，加上全部工件引用行数和工件文件数。""" + async with SessionLocal() as session: + runs = [] + for run_id in run_ids: + run = await session.get(Run, run_id) + count = await session.scalar(select(func.count()).select_from(RunEvent).where(RunEvent.run_id == run_id)) + runs.append((run.last_seq, run.manifest_hash, run.manifest_seq, run.status, count)) + artifacts = await session.scalar(select(func.count()).select_from(Artifact)) + files = sum(1 for _ in (settings.data_dir / "artifacts").rglob("*.json")) + return runs, artifacts, files + + +async def get(client, path: str, status: int = 200) -> dict: + resp = await client.get(path) + assert resp.status_code == status, resp.text + return resp.json() + + +# -------------------------------------------------------------------------- +# 对比 +# -------------------------------------------------------------------------- + + +async def test_compares_two_sealed_runs(client): + _, base_id, target_id = await pair() + body = await get(client, f"/api/runs/{target_id}/compare?base={base_id}") + assert body["target"]["id"] == target_id and body["base"]["id"] == base_id + assert set(body["target"]) == {"id", "workflow_name", "version", "run_class", "started_at", "finished_at", + "schedule_id"} + assert body["target"]["finished_at"].endswith("+00:00") and body["base"]["run_class"] == "formal" + assert body["summary"]["data_only"] == 2 and sum(body["summary"].values()) == 2 + east = next(m for m in body["metrics"] if m["metric_id"] == "east") + assert (east["classification"], east["base"], east["target"], east["delta_rendered"]) == ( + "data_only", 1200.5, 1500.5, "+300.0元") + total = next(m for m in body["metrics"] if m["metric_id"] == "total") + assert [r["share"] for r in total["attribution"]["rows"]] == [1.5, -0.5, 0.0] + assert [(q["node_id"], q["structure"]) for q in body["queries"]] == [("fetch", "same"), ("bot", "same")] + assert body["template"] == {"base_version": 3, "target_version": 3, "same_graph": True, "nodes": []} + assert body["notes"] == [] + + +async def test_comparing_writes_nothing(client): + _, base_id, target_id = await pair() + before = await footprint(base_id, target_id) + await get(client, f"/api/runs/{target_id}/compare?base={base_id}") + await get(client, f"/api/runs/{target_id}/compare/base") + assert await footprint(base_id, target_id) == before + for run_id in (base_id, target_id): + assert (await verify_manifest(run_id))["ok"] is True + + +async def test_unsealed_runs_are_refused(client): + workflow = await new_workflow() + base_id = await persist(week("r1", source=SOURCE), workflow) + target_id = await persist(week("r2", NEXT_ROWS, source=SOURCE), workflow, finished=T0 + timedelta(days=7), + seal=False) + body = await get(client, f"/api/runs/{target_id}/compare?base={base_id}", 409) + assert body["code"] == "compare_unsealed" and body["detail"].startswith("这次运行") + body = await get(client, f"/api/runs/{base_id}/compare?base={target_id}", 409) + assert body["code"] == "compare_unsealed" and body["detail"].startswith("作为基准的运行") + # 封存过、又被接着运行（还在运行）的：封存范围之后还会变，同样不比 + running = await persist(week("r3", NEXT_ROWS, source=SOURCE), workflow, status="running") + body = await get(client, f"/api/runs/{running}/compare?base={base_id}", 409) + assert body["code"] == "compare_unsealed" + + +async def test_a_broken_seal_is_refused(client): + _, base_id, target_id = await pair() + async with SessionLocal() as session: + event = (await session.execute(select(RunEvent).where( + RunEvent.run_id == base_id, RunEvent.type == "tool.end"))).scalars().first() + event.data = {**event.data, "duration_ms": 999} + await session.commit() + body = await get(client, f"/api/runs/{target_id}/compare?base={base_id}", 409) + assert body["code"] == "compare_seal_broken" and body["detail"].startswith("作为基准的运行") + assert "封存核对不一致" in body["detail"] + + +async def test_mismatched_runs_are_refused(client): + _, base_id, target_id = await pair() + other = await persist(week("r9", source=SOURCE), await new_workflow("另一个周报")) + body = await get(client, f"/api/runs/{target_id}/compare?base={other}", 422) + assert body["code"] == "compare_mismatch" and "同一个工作流" in body["detail"] + body = await get(client, f"/api/runs/{target_id}/compare?base={target_id}", 422) + assert body["code"] == "compare_mismatch" and "同一次运行" in body["detail"] + unsaved = await persist(week("r8", source=SOURCE), None) + body = await get(client, f"/api/runs/{unsaved}/compare?base={base_id}", 422) + assert body["code"] == "compare_mismatch" + body = await get(client, f"/api/runs/{target_id}/compare", 422) + assert body["code"] == "compare_bad_request" + await get(client, f"/api/runs/no-such-run/compare?base={base_id}", 404) + body = await get(client, f"/api/runs/{target_id}/compare?base=no-such-run", 404) + assert "基准" in body["detail"] + + +async def test_masked_columns_never_leave_the_server(client): + """快照里记着查询当时遮了 refund，数据源现在又遮了 manager：两样的值都不出现在结果里。""" + columns = ["region", "manager", "gmv", "refund"] + types = {"region": "text", "manager": "text", "gmv": "number", "refund": "number"} + sql = "SELECT region, manager, SUM(amount) AS gmv, SUM(refund) AS refund FROM orders GROUP BY region, manager" + + def make(run_id, rows, expression, version): + run = Fake(run_id, source=SOURCE) + fetch = run.tool("fetch", sql, columns, rows, types=types, masks=["refund"]) + bot = run.agent("bot", sql, columns, rows, types=types, masks=["refund"]) + inputs = [cell_in(0, "gmv", rows[0][2], fetch)] + value = rows[0][2] + if "refund" in expression: + inputs.append(cell_in(0, "refund", rows[0][3], fetch)) + value = rows[0][2] - rows[0][3] + run.card([metric("net", expression, value, inputs), + metric("total", "sum(vars.kpi.amounts)", sum(r[2] for r in rows), [amounts_in(bot, rows)])], + version=version) + return run + + async with SessionLocal() as session: + session.add(DataSource(name=SOURCE, kind="sqlite", database=":memory:", options={"mask_columns": ["manager"]})) + await session.commit() + try: + base = make("r1", [["东区", "经理甲", 1200.5, 101.25], ["南区", "经理乙", 800.0, 52.75]], EAST, "v1") + target = make("r2", [["东区", "经理甲", 1500.5, 133.5], ["南区", "经理乙", 700.0, 47.25]], + f"{EAST} - cell(nodes.fetch, 0, 'refund')", "v2") + _, base_id, target_id = await pair(base, target) + resp = await client.get(f"/api/runs/{target_id}/compare?base={base_id}") + assert resp.status_code == 200, resp.text + for secret in ("经理甲", "经理乙", "101.25", "52.75", "133.5", "47.25"): + assert secret not in resp.text, secret + body = resp.json() + net = next(m for m in body["metrics"] if m["metric_id"] == "net") + refund = next(i for i in net["inputs"] if "refund" in i["path"]) + assert refund["masked"] is True and net["decomposition"] is None and "遮罩" in net["decomposition_note"] + total = next(m for m in body["metrics"] if m["metric_id"] == "total") + assert total["attribution"]["columns"] == ["region"] + finally: + await drop_source(name=SOURCE) + + +# -------------------------------------------------------------------------- +# 默认基准 +# -------------------------------------------------------------------------- + + +async def bare(workflow_id, finished, *, status="succeeded", run_class="formal", schedule_id=None, sealed=True) -> str: + """只为挑基准用的运行记录：挑基准只看运行记录上的几列。""" + async with SessionLocal() as session: + run = Run(workflow_id=workflow_id, workflow_name="区域周报", status=status, run_class=run_class, + schedule_id=schedule_id, started_at=finished - timedelta(minutes=2), + finished_at=finished if status in ("succeeded", "failed", "cancelled") else None, + manifest_hash="ab" * 32 if sealed else None, manifest_seq=1 if sealed else None) + session.add(run) + await session.commit() + return run.id + + +async def test_the_default_base_prefers_the_same_schedule(client): + workflow = await new_workflow() + day = timedelta(days=1) + scheduled = await bare(workflow, T0, schedule_id="sched-weekly") + manual = await bare(workflow, T0 + day) # 更近，但不是同一个定时任务 + current = await bare(workflow, T0 + 2 * day, schedule_id="sched-weekly") + body = await get(client, f"/api/runs/{current}/compare/base") + assert (body["base_run_id"], body["reason"]) == (scheduled, "same_schedule") + assert body["base"]["id"] == scheduled and body["base"]["schedule_id"] == "sched-weekly" + # 不是定时任务发起的：同一工作流、同一运行类别里最近的一次（定时的也算） + later = await bare(workflow, T0 + 3 * day) + body = await get(client, f"/api/runs/{later}/compare/base") + assert (body["base_run_id"], body["reason"]) == (current, "same_workflow") + # 定时任务的第一次：退回同一工作流、同一运行类别 + first = await bare(workflow, T0 + 4 * day, schedule_id="sched-daily") + body = await get(client, f"/api/runs/{first}/compare/base") + assert (body["base_run_id"], body["reason"]) == (later, "same_workflow") + assert manual != body["base_run_id"] + + +async def test_the_default_base_skips_what_cannot_be_compared(client): + workflow = await new_workflow() + day = timedelta(days=1) + good = await bare(workflow, T0) + await bare(workflow, T0 + day, sealed=False) # 没封存 + await bare(workflow, T0 + 2 * day, status="running") # 还在运行 + await bare(workflow, T0 + 3 * day, run_class="exploratory") # 别的运行类别 + await bare(await new_workflow("另一个周报"), T0 + 4 * day) # 别的工作流 + current = await bare(workflow, T0 + 5 * day) + await bare(workflow, T0 + 6 * day) # 比它晚结束 + body = await get(client, f"/api/runs/{current}/compare/base") + assert (body["base_run_id"], body["reason"]) == (good, "same_workflow") + # 失败、取消的终态运行也算（封存过） + failed = await bare(workflow, T0 + 5 * day - timedelta(hours=1), status="failed") + assert (await get(client, f"/api/runs/{current}/compare/base"))["base_run_id"] == failed + # 第一次运行、没保存的工作流：没有可比的 + body = await get(client, f"/api/runs/{good}/compare/base", 404) + assert body["code"] == "compare_no_base" and "上一次运行" in body["detail"] + await get(client, f"/api/runs/{await bare(None, T0)}/compare/base", 404) + await get(client, "/api/runs/no-such-run/compare/base", 404) + + +# -------------------------------------------------------------------------- +# 真跑：虚构的 SQLite 库，两次运行之间改了数、换了日期参数 +# -------------------------------------------------------------------------- + +E2E_SOURCE = "cmp_e2e" +E2E_SQL = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE day BETWEEN '{{ input.start }}' AND '{{ input.end }}' " + "GROUP BY region ORDER BY region") +WEEK36_INPUT = {"start": "2026-08-31", "end": "2026-09-06"} +WEEK37_INPUT = {"start": "2026-09-07", "end": "2026-09-13"} + + +@pytest.fixture +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def orders(tmp_path): + path = tmp_path / "orders.db" + db = sqlite3.connect(path) + db.executescript("CREATE TABLE orders (id INTEGER PRIMARY KEY, day TEXT, region TEXT, amount REAL);") + db.executemany("INSERT INTO orders (day, region, amount) VALUES (?, ?, ?)", [ + ("2026-09-01", "东区", 1200.5), ("2026-09-02", "南区", 800.0), ("2026-09-03", "西区", 600.0), + ("2026-09-08", "东区", 900.0), ("2026-09-09", "南区", 650.0), ("2026-09-10", "西区", 640.0)]) + db.commit() + db.close() + async with SessionLocal() as session: + row = DataSource(name=E2E_SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + await session.commit() + source_id = row.id + await engines.invalidate(source_id) + yield path, source_id + await drop_source(source_id) + + +def e2e_graph(): + return chain( + node("start", "input", "开始"), + node("fetch", "tool", "区域取数", tool=f"db_query__{E2E_SOURCE}", args={"sql": E2E_SQL}), + node("card", "metrics", "周报口径卡", caliber="周报口径", caliber_version="v1", metrics=[ + {"id": "east", "name": "东区金额", "unit": "元", "decimals": 1, "format": "thousands", + "expression": "cell(nodes.fetch, 0, 'gmv')"}]), + node("out", "output", "成果", fields=[{"name": "指标", "value": "{{ nodes.card.text }}"}]), + ) + + +async def finished_run(workflow_id: str, inputs: dict) -> str: + run = await run_manager.start(graph=e2e_graph(), input_payload=inputs, workflow_id=workflow_id, + workflow_name="区域周报") + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + assert row.status == "succeeded", row.error + return run.id + raise AssertionError(f"运行 {run.id} 没有结束：{row.status}") + + +async def test_real_runs_split_data_and_parameter_changes(client, engine_up, orders): + path, source_id = orders + workflow = await new_workflow() + first = await finished_run(workflow, WEEK36_INPUT) + db = sqlite3.connect(path) + db.execute("UPDATE orders SET amount = 1500.5 WHERE day = '2026-09-01'") + db.commit() + db.close() + await engines.invalidate(source_id) + second = await finished_run(workflow, WEEK36_INPUT) + third = await finished_run(workflow, WEEK37_INPUT) + + body = await get(client, f"/api/runs/{second}/compare?base={first}") + [east] = body["metrics"] + assert (east["classification"], east["base"], east["target"], east["delta"]) == ("data_only", 1200.5, 1500.5, 300.0) + [fetch] = body["queries"] + assert (fetch["node_id"], fetch["structure"], fetch["base_rows"], fetch["target_rows"]) == ("fetch", "same", 3, 3) + assert east["inputs"][0]["base_artifact"] == fetch["base_artifact"] + # 东区金额只是三个区域里的一格：各区域的变化照列，占比为空 + assert [(r["key"], r["delta"]) for r in east["attribution"]["rows"]] == [ + (["东区"], 300.0), (["南区"], 0), (["西区"], 0)] + + body = await get(client, f"/api/runs/{third}/compare?base={second}") + [east] = body["metrics"] + assert east["classification"] == "params_only" and east["target"] == 900.0 + [fetch] = body["queries"] + assert fetch["structure"] == "params_only" + assert fetch["literals"] == [{"base": "'2026-08-31'", "target": "'2026-09-07'"}, + {"base": "'2026-09-06'", "target": "'2026-09-13'"}] + assert body["template"]["same_graph"] is True + assert any("start" in n for n in body["notes"]), body["notes"] + + body = await get(client, f"/api/runs/{third}/compare/base") + assert (body["base_run_id"], body["reason"]) == (second, "same_workflow") + json.dumps(body, allow_nan=False) diff --git a/backend/tests/test_compare_engine.py b/backend/tests/test_compare_engine.py new file mode 100644 index 00000000..2b7a49c7 --- /dev/null +++ b/backend/tests/test_compare_engine.py @@ -0,0 +1,558 @@ +"""运行对比的纯函数（engine/compare.py）：同一个工作流的两次运行，每个指标的变化拆成数据、口径、模板 / SQL。 + +两次运行都是照真实运行的事件形状手写的虚构数据（区域、金额、日期都是编的）：取数节点 fetch 查一条按区域汇总的 +SQL，Agent 节点 bot 自己再查一次、把各区域的金额作为数组字段交给口径卡，口径卡算几个指标。工件放在字典里。守的是： +- 只数据变（SQL 一字不差、结果变了）是 data_only；只改日期字面量是 params_only；SQL 结构变了是 sql_changed、不拆 +- 口径变了（表达式、口径版本）是 caliber_and_data：旧表达式的叶子路径在这次的输入里都取得到时严格拆分， + 口径效应 + 数据效应 = 总变化（逐位相等）；取不到就不拆，说明缺的是哪一个 +- 维度归因：按维度键配对两边的行，按变化的绝对值取前 10；度量列之和等于输入的值（两边都等）才给占比，占比之和 + 为 1；对不上时占比为空并说明；截断的快照不归因，照实写 +- 遮罩列的值绝不出现在结果里；用到遮罩列的指标不拆分 +- 模板对比：新增、删除的节点，配置有变化的节点和配置项名 +""" +from __future__ import annotations + +import json +from decimal import Decimal + +from app.engine import compare +from tests.fixtures.compare_runs import ( + BASE_ROWS, + EAST, + NEXT_ROWS, + REFUND, + WEEK37, + Fake, + amounts_in, + cell_in, + default_metrics, + metric, + node, + region_sql, + scalar_in, + week, + weekly_graph, +) + + +def run_compare(base, target): + result = compare.compare(base.side(), target.side()) + json.dumps(result, allow_nan=False) # 结果要能原样交给接口 + return result + + +def by_id(result): + return {m["metric_id"]: m for m in result["metrics"]} + + +def exact(value): + return Decimal(repr(value)) if isinstance(value, float) else Decimal(value) + + +# -------------------------------------------------------------------------- +# 分类：只数据变、只参数变、SQL 结构变 +# -------------------------------------------------------------------------- + + +def test_same_sql_new_rows_is_data_only(): + result = run_compare(week("r1"), week("r2", NEXT_ROWS)) + metrics = by_id(result) + east, total = metrics["east"], metrics["total"] + assert (east["classification"], total["classification"]) == ("data_only", "data_only") + assert (east["base"], east["target"], east["delta"]) == (1200.5, 1500.5, 300.0) + assert (east["base_rendered"], east["target_rendered"], east["delta_rendered"]) == ("1,200.5元", "1,500.5元", "+300.0元") + assert east["delta_pct"] == round(300 / 1200.5 * 100, 2) + assert east["card_id"] == "card" and east["card_title"] == "周报口径卡" and east["name"] == "东区金额" + assert east["decomposition"] is None and east["decomposition_note"] is None and east["reasons"] + assert total["delta"] == 200.0 and total["delta_rendered"] == "+200.0元" + assert result["summary"] == {"unchanged": 0, "data_only": 2, "caliber_and_data": 0, "sql_changed": 0, + "params_only": 0, "added": 0, "removed": 0, "incomparable": 0} + # 输入两边的值都列出来，带着各自的查询快照 + [cell] = east["inputs"] + assert (cell["path"], cell["base"], cell["target"]) == (EAST, 1200.5, 1500.5) + assert cell["base_artifact"] != cell["target_artifact"] + # 查询：节点 + 第几次 + 工具配上对，SQL 一字不差 + assert [(q["node_id"], q["ordinal"], q["structure"]) for q in result["queries"]] == [ + ("fetch", 1, "same"), ("bot", 1, "same")] + fetch = result["queries"][0] + assert fetch["node_title"] == "区域取数" and fetch["base_sql"] == fetch["target_sql"] == region_sql() + assert fetch["literals"] == [] and (fetch["base_rows"], fetch["target_rows"]) == (3, 3) + assert (fetch["base_truncated"], fetch["target_truncated"]) == (False, False) + assert result["template"]["same_graph"] is True and result["template"]["nodes"] == [] + assert (result["base"]["id"], result["target"]["id"]) == ("r1", "r2") + + +def test_identical_runs_are_unchanged(): + result = run_compare(week("r1"), week("r2")) + assert {m["classification"] for m in result["metrics"]} == {"unchanged"} + east = by_id(result)["east"] + assert east["delta"] == 0 and east["delta_rendered"] is None and east["attribution"] is None + assert result["summary"]["unchanged"] == 2 + + +def test_only_date_literals_changed_is_params_only(): + # 图里的 SQL 是模板（两次一样），执行时代进去的日期不同 + graph = weekly_graph(region_sql(("{{ input.start }}", "{{ input.end }}"))) + result = run_compare(week("r1", graph=graph), week("r2", NEXT_ROWS, sql=region_sql(WEEK37), graph=graph)) + metrics = by_id(result) + assert {m["classification"] for m in metrics.values()} == {"params_only"} + assert any("参数" in r for r in metrics["east"]["reasons"]) + fetch = result["queries"][0] + assert fetch["structure"] == "params_only" + assert fetch["literals"] == [{"base": "'2026-08-31'", "target": "'2026-09-07'"}, + {"base": "'2026-09-06'", "target": "'2026-09-13'"}] + # 定时运行换了日期属于参数变化：模板（图）一样 + assert result["summary"]["params_only"] == 2 and result["template"]["same_graph"] is True + + +def test_a_changed_sql_structure_is_flagged_not_split(): + paid = region_sql(extra=" AND status = 'paid'") + result = run_compare(week("r1"), week("r2", NEXT_ROWS, sql=paid)) + metrics = by_id(result) + assert {m["classification"] for m in metrics.values()} == {"sql_changed"} + east = metrics["east"] + assert east["decomposition"] is None and east["attribution"] is None + assert any("区域取数" in r and "SQL" in r for r in east["reasons"]) + fetch = result["queries"][0] + assert fetch["structure"] == "changed" and fetch["literals"] == [] + assert any("status = 'paid'" in d for d in fetch["differences"]) + # 图里 fetch 的 SQL 也改了：模板对比列出它和改了的配置项 + [changed] = result["template"]["nodes"] + assert (changed["node_id"], changed["change"], changed["keys"]) == ("fetch", "config", ["args.sql"]) + assert result["template"]["same_graph"] is False + + +# -------------------------------------------------------------------------- +# 口径变了：严格拆分 +# -------------------------------------------------------------------------- + + +def net_metrics(expression): + """净额指标 net：表达式由调用方定（v1 只取金额，v2 减去退款）。""" + def build(fetch, bot, rows, bot_rows): + gmv, refund = rows[0][1], rows[0][2] + inputs = [cell_in(0, "gmv", gmv, fetch)] + value = gmv + if REFUND in expression: + inputs.append(cell_in(0, "refund", refund, fetch)) + value = gmv - refund + return [metric("net", expression, value, inputs, name="东区净额")] + return build + + +def test_a_caliber_change_is_split_into_caliber_and_data_effects(): + base = week("r1", metrics=net_metrics(EAST), version="v1") + target = week("r2", NEXT_ROWS, metrics=net_metrics(f"{EAST} - {REFUND}"), version="v2") + net = by_id(run_compare(base, target))["net"] + assert net["classification"] == "caliber_and_data" + assert any("表达式" in r for r in net["reasons"]) and any("v1" in r and "v2" in r for r in net["reasons"]) + assert (net["base"], net["target"], net["delta"]) == (1200.5, 1380.5, 180.0) + split = net["decomposition"] + assert net["decomposition_note"] is None + # 口径效应 = f_new(x_new) − f_old(x_new) = 1380.5 − 1500.5；数据效应 = f_old(x_new) − f_old(x_old) = 1500.5 − 1200.5 + assert (split["caliber_effect"], split["data_effect"]) == (-120.0, 300.0) + assert (split["caliber_rendered"], split["data_rendered"]) == ("-120.0元", "+300.0元") + assert exact(split["caliber_effect"]) + exact(split["data_effect"]) == exact(net["delta"]) + + +def test_the_split_adds_up_exactly_with_awkward_decimals(): + """小数位不同、浮点相减有误差的数：两效应之和逐位等于总变化。""" + def build(expression, decimals): + def make(fetch, bot, rows, bot_rows): + gmv, refund = rows[0][1], rows[0][2] + inputs = [cell_in(0, "gmv", gmv, fetch), cell_in(0, "refund", refund, fetch)] + value = round(eval(expression.replace(EAST, repr(gmv)).replace(REFUND, repr(refund))), decimals) + return [metric("ratio", expression, value, inputs, decimals=decimals, unit="")] + return make + + base_rows = [["东区", 1000.1, 0.3], *BASE_ROWS[1:]] + next_rows = [["东区", 1000.7, 0.2], *NEXT_ROWS[1:]] + base = week("r1", base_rows, metrics=build(f"{EAST} * 0.1 - {REFUND}", 2)) + target = week("r2", next_rows, metrics=build(f"{EAST} * 0.3 + {REFUND}", 3), version="v2") + ratio = by_id(run_compare(base, target))["ratio"] + split = ratio["decomposition"] + assert split is not None, ratio["decomposition_note"] + # f_old(x_new) 按旧口径的小数位取整：round(1000.7 * 0.1 - 0.2, 2) = 99.87 + assert exact(split["data_effect"]) == exact(99.87) - exact(ratio["base"]) + assert exact(split["caliber_effect"]) + exact(split["data_effect"]) == exact(ratio["delta"]) + assert exact(ratio["delta"]) == exact(ratio["target"]) - exact(ratio["base"]) + + +def test_a_version_bump_with_the_same_formula_is_all_data(): + base = week("r1", metrics=net_metrics(EAST), version="v1") + target = week("r2", NEXT_ROWS, metrics=net_metrics(EAST), version="v2") + net = by_id(run_compare(base, target))["net"] + assert net["classification"] == "caliber_and_data" + assert (net["decomposition"]["caliber_effect"], net["decomposition"]["data_effect"]) == (0, 300.0) + assert net["decomposition"]["caliber_rendered"] == "0.0元" + + +def test_no_split_when_an_old_leaf_is_not_recorded_this_time(): + base = week("r1", metrics=net_metrics(f"{EAST} - {REFUND}"), version="v1") + target = week("r2", NEXT_ROWS, metrics=net_metrics(EAST), version="v2") + net = by_id(run_compare(base, target))["net"] + assert net["classification"] == "caliber_and_data" and net["decomposition"] is None + assert REFUND in net["decomposition_note"] and "无法严格拆分" in net["decomposition_note"] + # 总变化照算 + assert net["delta"] == 1500.5 - (1200.5 - 100.0) + + +def test_no_split_when_the_sql_changed_too(): + base = week("r1", metrics=net_metrics(EAST), version="v1") + target = week("r2", NEXT_ROWS, sql=region_sql(extra=" AND status = 'paid'"), metrics=net_metrics(EAST), version="v2") + net = by_id(run_compare(base, target))["net"] + assert net["classification"] == "caliber_and_data" and net["decomposition"] is None + assert "SQL" in net["decomposition_note"] + + +def test_a_unit_change_is_not_subtracted(): + def build(unit): + def make(fetch, bot, rows, bot_rows): + return [metric("east", EAST, rows[0][1], [cell_in(0, "gmv", rows[0][1], fetch)], unit=unit)] + return make + east = by_id(run_compare(week("r1", metrics=build("元")), week("r2", NEXT_ROWS, metrics=build("万元"))))["east"] + assert east["classification"] == "caliber_and_data" and any("单位" in r for r in east["reasons"]) + assert east["delta"] is None and east["delta_rendered"] is None and east["decomposition"] is None + assert "单位" in east["decomposition_note"] + + +# -------------------------------------------------------------------------- +# 维度归因 +# -------------------------------------------------------------------------- + + +def test_attribution_pairs_keys_and_shares_add_up_to_one(): + total = by_id(run_compare(week("r1"), week("r2", NEXT_ROWS)))["total"] + attribution = total["attribution"] + assert attribution["columns"] == ["region"] and attribution["measure"] == "gmv" + assert attribution["total_delta"] == 200.0 + rows = attribution["rows"] + assert [(r["key"], r["base"], r["target"], r["delta"]) for r in rows] == [ + (["东区"], 1200.5, 1500.5, 300.0), (["南区"], 800.0, 700.0, -100.0), (["西区"], 600.0, 600.0, 0.0)] + assert [r["share"] for r in rows] == [1.5, -0.5, 0.0] + assert abs(sum(r["share"] for r in rows) - 1) < 1e-12 + + +def test_attribution_pairs_by_key_not_by_row_position(): + """这次多了一个区域、顺序也变了：按区域配对，只有一边有的那一边写空。""" + next_rows = [["北区", 90.0, 0.0], ["东区", 1500.5, 120.0], ["西区", 600.0, 10.0], ["南区", 700.0, 40.0]] + total = by_id(run_compare(week("r1"), week("r2", next_rows)))["total"] + rows = {tuple(r["key"]): r for r in total["attribution"]["rows"]} + assert rows[("北区",)]["base"] is None and rows[("北区",)]["target"] == 90.0 and rows[("北区",)]["delta"] == 90.0 + assert rows[("东区",)]["delta"] == 300.0 and rows[("南区",)]["delta"] == -100.0 + assert abs(sum(r["share"] for r in rows.values()) - 1) < 1e-12 + + +def test_attribution_keeps_the_ten_largest_changes(): + names = [f"{n}区" for n in "一二三四五六七八九十甲乙"] + base_rows = [[name, 100.0, 0.0] for name in names] + next_rows = [[name, 100.0 + (i + 1) * 10 * (-1) ** i, 0.0] for i, name in enumerate(names)] + total = by_id(run_compare(week("r1", base_rows), week("r2", next_rows)))["total"] + rows = total["attribution"]["rows"] + assert len(rows) == 10 + assert [abs(r["delta"]) for r in rows] == [120.0, 110.0, 100.0, 90.0, 80.0, 70.0, 60.0, 50.0, 40.0, 30.0] + assert "12" in total["attribution"]["note"] + # 占比按全部维度值的总变化算：前 10 个之和不必为 1，全部之和为 1 + assert total["attribution"]["total_delta"] == sum((i + 1) * 10 * (-1) ** i for i in range(12)) + + +def test_no_share_when_the_measure_does_not_add_up_to_the_input(): + """东区金额只是快照里的一格：各区域之和不等于它，各键的变化照列，占比为空并说明。""" + east = by_id(run_compare(week("r1"), week("r2", NEXT_ROWS)))["east"] + attribution = east["attribution"] + assert attribution["measure"] == "gmv" and [r["key"] for r in attribution["rows"]] == [["东区"], ["南区"], ["西区"]] + assert all(r["share"] is None for r in attribution["rows"]) and attribution["note"] + + +def test_a_total_row_is_left_out_of_its_own_breakdown(): + """合计行和各区域在同一份结果里：指标取合计那一格，按其余各行归因，占比之和为 1。""" + base_rows = [*BASE_ROWS, ["合计", 2600.5, 150.0]] + next_rows = [*NEXT_ROWS, ["合计", 2800.5, 170.0]] + + def build(fetch, bot, rows, bot_rows): + return [metric("all", "cell(nodes.fetch, 3, 'gmv')", rows[3][1], [cell_in(3, "gmv", rows[3][1], fetch)])] + total = by_id(run_compare(week("r1", base_rows, metrics=build), week("r2", next_rows, metrics=build)))["all"] + rows = total["attribution"]["rows"] + assert ["合计"] not in [r["key"] for r in rows] and len(rows) == 3 + assert abs(sum(r["share"] for r in rows) - 1) < 1e-12 and total["attribution"]["total_delta"] == 200.0 + + +def test_a_truncated_snapshot_is_not_attributed(): + base, target = week("r1"), Fake("r2") + sql = region_sql() + fetch = target.tool("fetch", sql, rows=NEXT_ROWS) + bot = target.agent("bot", sql, rows=NEXT_ROWS, truncated=True) + target.card(default_metrics(fetch, bot, NEXT_ROWS, NEXT_ROWS)) + total = by_id(run_compare(base, target))["total"] + attribution = total["attribution"] + assert attribution["rows"] == [] and "截断" in attribution["note"] + assert result_queries(run_compare(base, target), "bot")["target_truncated"] is True + + +def result_queries(result, node_id): + return next(q for q in result["queries"] if q["node_id"] == node_id) + + +def test_a_period_column_that_never_matches_is_not_a_dimension(): + """每期都不同的列（这里是日期写成的文本）不拿来配对：按其余的维度列归因。""" + columns = ["day", "region", "gmv"] + types = {"day": "text", "region": "text", "gmv": "number"} + + def make(run_id, days, values): + run = Fake(run_id) + rows = [[day, region, value] for day in days for region, value in zip(("东区", "南区"), values)] + sql = "SELECT day, region, SUM(amount) AS gmv FROM orders GROUP BY day, region" + bot = run.agent("bot", sql, columns, rows, types=types) + total = sum(r[2] for r in rows) + run.card([metric("total", "sum(vars.kpi.amounts)", total, [amounts_in(bot, rows)])]) + return run + + total = by_id(run_compare(make("r1", ("2026-09-05", "2026-09-06"), (10.0, 20.0)), + make("r2", ("2026-09-12", "2026-09-13"), (15.0, 18.0))))["total"] + attribution = total["attribution"] + assert attribution["columns"] == ["region"] + assert [(r["key"], r["delta"]) for r in attribution["rows"]] == [(["东区"], 10.0), (["南区"], -4.0)] + assert "day" in attribution["note"] + + +# -------------------------------------------------------------------------- +# 遮罩列 +# -------------------------------------------------------------------------- + + +def test_masked_columns_never_show_up(): + """数据源遮了 manager（经理名）和 refund：经理名不当维度、不出现在结果里；用到 refund 的指标输入不给值、不拆分。""" + columns = ["region", "manager", "gmv", "refund"] + types = {"region": "text", "manager": "text", "gmv": "number", "refund": "number"} + + def make(run_id, rows, expression, version): + run = Fake(run_id) + sql = "SELECT region, manager, SUM(amount) AS gmv, SUM(refund) AS refund FROM orders GROUP BY region, manager" + fetch = run.tool("fetch", sql, columns, rows, types=types) + bot = run.agent("bot", sql, columns, rows, types=types) + run.masked = {fetch: frozenset({"manager", "refund"}), bot: frozenset({"manager", "refund"})} + inputs = [cell_in(0, "gmv", rows[0][2], fetch)] + value = rows[0][2] + if "refund" in expression: + inputs.append(cell_in(0, "refund", rows[0][3], fetch)) + value = rows[0][2] - rows[0][3] + run.card([metric("net", expression, value, inputs), + metric("total", "sum(vars.kpi.amounts)", sum(r[2] for r in rows), [amounts_in(bot, rows)])], + version=version) + return run + + base_rows = [["东区", "经理甲", 1200.5, 101.25], ["南区", "经理乙", 800.0, 52.75]] + next_rows = [["东区", "经理甲", 1500.5, 133.5], ["南区", "经理乙", 700.0, 47.25]] + result = run_compare(make("r1", base_rows, EAST, "v1"), + make("r2", next_rows, "cell(nodes.fetch, 0, 'gmv') - cell(nodes.fetch, 0, 'refund')", "v2")) + text = json.dumps(result, ensure_ascii=False) + for secret in ("经理甲", "经理乙", "101.25", "52.75", "133.5", "47.25"): + assert secret not in text, secret + metrics = by_id(result) + net = metrics["net"] + refund = next(i for i in net["inputs"] if "refund" in i["path"]) + assert refund["masked"] is True and refund["base"] is None and refund["target"] is None + assert net["decomposition"] is None and "遮罩" in net["decomposition_note"] + # 维度只用没遮的 region；占比照给 + attribution = metrics["total"]["attribution"] + assert attribution["columns"] == ["region"] and abs(sum(r["share"] for r in attribution["rows"]) - 1) < 1e-12 + assert any("遮罩" in n for n in result["notes"]) + + +def test_a_masked_measure_is_not_attributed(): + def masked(run): + run.masked = {a: frozenset({"gmv"}) for a in run.queries} + return run + result = run_compare(masked(week("r1")), masked(week("r2", NEXT_ROWS))) + total = by_id(result)["total"] + assert total["attribution"]["rows"] == [] and "遮罩" in total["attribution"]["note"] + east = by_id(result)["east"] + assert east["inputs"][0]["base"] is None and east["inputs"][0]["masked"] is True + assert "1200.5" not in json.dumps(east["inputs"]) and "800.0" not in json.dumps(result) + + +# -------------------------------------------------------------------------- +# 对不上具体查询的输入、运行输入、一边才有的指标 +# -------------------------------------------------------------------------- + + +def code_metrics(value): + def build(fetch, bot, rows, bot_rows): + return [metric("calc", "vars.adj * 2", value * 2, [scalar_in("vars.adj", value, "calc", "code")])] + return build + + +def test_an_input_from_a_code_node_follows_its_upstream(): + def make(run_id, rows, value, sql=None): + run = Fake(run_id, graph=weekly_graph(sql, between=(node("calc", "code", "调整", code="print(1)", + assign_to="adj"),))) + sql = sql or region_sql() + run.tool("fetch", sql, rows=rows) + run.agent("bot", sql, rows=rows) + run.code("calc", value, "adj") + run.card(code_metrics(value)(None, None, rows, rows)) + return run + + # 上游什么都没变：变化来自数据 + calc = by_id(run_compare(make("r1", BASE_ROWS, 10), make("r2", NEXT_ROWS, 12)))["calc"] + assert calc["classification"] == "data_only" and calc["delta"] == 4 + # 上游的 SQL 结构变了：说不清它变没变，不能报成数据变化 + calc = by_id(run_compare(make("r1", BASE_ROWS, 10), + make("r2", NEXT_ROWS, 12, sql=region_sql(extra=" AND status = 'paid'"))))["calc"] + assert calc["classification"] == "incomparable" and any("调整" in r for r in calc["reasons"]) + + +def test_the_code_node_itself_changed_is_a_template_change(): + def make(run_id, code, value): + run = Fake(run_id, graph=weekly_graph(between=(node("calc", "code", "调整", code=code, assign_to="adj"),))) + run.tool("fetch", region_sql()) + run.agent("bot", region_sql()) + run.code("calc", value, "adj") + run.card(code_metrics(value)(None, None, BASE_ROWS, BASE_ROWS)) + return run + calc = by_id(run_compare(make("r1", "x = 1", 10), make("r2", "x = 2", 11)))["calc"] + assert calc["classification"] == "sql_changed" and any("调整" in r for r in calc["reasons"]) + + +def test_a_run_input_change_is_a_parameter_change(): + def make(run_id, goal): + run = Fake(run_id, inputs={"goal": goal}) + run.add("node.started", "start", {"node_type": "input"}) + run.add("node.finished", "start", {"artifact": run.put({"goal": goal})}) + fetch = run.tool("fetch", region_sql()) + run.agent("bot", region_sql()) + run.card([metric("reach", f"round({EAST} / input.goal * 100, 1)", round(1200.5 / goal * 100, 1), + [cell_in(0, "gmv", 1200.5, fetch), scalar_in("input.goal", goal, "start", "input")], + unit="%", fmt="plain")]) + return run + result = run_compare(make("r1", 1000), make("r2", 1500)) + reach = by_id(result)["reach"] + assert reach["classification"] == "params_only" and any("input.goal" in r for r in reach["reasons"]) + assert reach["delta_rendered"] == "-40.0 个百分点" + assert any("goal" in n for n in result["notes"]) + + +def test_metrics_and_cards_on_one_side_only(): + def extra(fetch, bot, rows, bot_rows): + return [*default_metrics(fetch, bot, rows, bot_rows), + metric("west", "cell(nodes.fetch, 2, 'gmv')", rows[2][1], [cell_in(2, "gmv", rows[2][1], fetch)])] + base = week("r1") + target = week("r2", metrics=extra) + target.card([metric("cnt", "len(vars.kpi.amounts)", 3, [amounts_in(target.queries[1], BASE_ROWS)], unit="个", + decimals=None)], node_id="card2", caliber="补充口径") + result = run_compare(base, target) + metrics = by_id(result) + assert metrics["west"]["classification"] == "added" and metrics["west"]["base"] is None + assert metrics["cnt"]["classification"] == "added" and metrics["cnt"]["card_id"] == "card2" + assert result["summary"]["added"] == 2 + swapped = by_id(run_compare(target, base)) + assert swapped["west"]["classification"] == "removed" and swapped["west"]["target"] is None + + +def test_an_unreadable_card_makes_its_metrics_incomparable(): + base, target = week("r1"), week("r2", NEXT_ROWS) + base.tampered.add(base.cards[0][1]) + base.cards = [] # 接口只交回复验过哈希的口径卡 + result = run_compare(base, target) + assert {m["classification"] for m in result["metrics"]} == {"incomparable"} + assert any("哈希" in r for r in by_id(result)["east"]["reasons"]) + + +def test_a_card_executed_several_times_uses_the_last_one(): + base = week("r1") + target = Fake("r2") + fetch = target.tool("fetch", region_sql(), rows=NEXT_ROWS) + bot = target.agent("bot", region_sql(), rows=NEXT_ROWS) + target.card(default_metrics(fetch, bot, BASE_ROWS, BASE_ROWS)) + target.card(default_metrics(fetch, bot, NEXT_ROWS, NEXT_ROWS), runs=2) + result = run_compare(base, target) + assert by_id(result)["east"]["target"] == 1500.5 + assert any("执行了 3 次" in n for n in result["notes"]) + + +def test_verify_queries_are_left_out(): + base = week("r1") + target = Fake("r2") + fetch = target.tool("fetch", region_sql(), rows=NEXT_ROWS) + bot = target.agent("bot", region_sql(), rows=NEXT_ROWS, verify=["SELECT COUNT(*) AS n FROM orders"]) + target.card(default_metrics(fetch, bot, NEXT_ROWS, NEXT_ROWS)) + result = run_compare(base, target) + assert [(q["node_id"], q["structure"]) for q in result["queries"]] == [("fetch", "same"), ("bot", "same")] + + +# -------------------------------------------------------------------------- +# 查询配对、模板对比 +# -------------------------------------------------------------------------- + + +def test_queries_fall_back_to_normalized_sql_pairing(): + """节点换了 ID：按「节点 + 第几次 + 工具」配不上，再按规范化 SQL 配（大小写、空白不同也认）。""" + base = week("r1") + target = Fake("r2", graph=weekly_graph()) + target.tool("pull", region_sql(WEEK37).lower(), rows=NEXT_ROWS) + target.agent("bot", region_sql(WEEK37), rows=NEXT_ROWS) + result = run_compare(base, target) + pull = result_queries(result, "pull") + assert pull["structure"] == "params_only" and pull["base_sql"] == region_sql() + assert len(result["queries"]) == 2 + + +def test_unpaired_queries_are_added_or_removed(): + base = week("r1") + target = week("r2") + target.tool("extra", "SELECT COUNT(*) AS n FROM refunds", ["n"], [[7]], types={"n": "number"}) + result = run_compare(base, target) + extra = result_queries(result, "extra") + assert extra["structure"] == "added" and extra["base_sql"] is None and extra["target_rows"] == 1 + removed = result_queries(run_compare(target, base), "extra") + assert removed["structure"] == "removed" and removed["target_sql"] is None + + +def test_the_template_lists_added_removed_and_reconfigured_nodes(): + base_graph = weekly_graph(between=(node("calc", "code", "调整", code="x = 1"),)) + target_graph = weekly_graph(region_sql(WEEK37), between=(node("check", "dual_check", "双盲复算"),)) + target_graph["nodes"][-2]["data"]["label"] = "写周报（新）" + target_graph["nodes"][-1]["data"]["config"] = {"fields": [{"name": "周报", "value": "{{ nodes.write.text }}"}]} + base = week("r1", graph=base_graph) + target = week("r2", graph=target_graph, version="v1") + base.version, target.version = 3, 4 + template = run_compare(base, target)["template"] + assert (template["base_version"], template["target_version"], template["same_graph"]) == (3, 4, False) + changes = {n["node_id"]: (n["change"], n["keys"]) for n in template["nodes"]} + assert changes == {"fetch": ("config", ["args.sql"]), "check": ("added", []), "write": ("config", ["label"]), + "out": ("config", ["fields"]), "calc": ("removed", [])} + labels = {n["node_id"]: n["labels"] for n in template["nodes"]} + assert labels["write"] == ["节点名称"] and labels["out"] == ["成果字段"] + # 配置的值（这里是 SQL 和模板）一个字都不展示 + assert "2026-09" not in json.dumps(template, ensure_ascii=False) + + +def test_a_missing_snapshot_file_is_reported_and_withheld(): + """基准运行的快照文件不在了：SQL 从 tool.start 的参数里取，结构照比；输入的值核对不了遮没遮，不给；不归因。""" + base, target = week("r1"), week("r2", NEXT_ROWS) + fetch = base.queries[0] + del base.store[fetch] + result = run_compare(base, target) + query = result_queries(result, "fetch") + assert query["structure"] == "same" and query["base_sql"] == region_sql() and query["base_rows"] is None + assert "无法读取" in query["problem"] + east = by_id(result)["east"] + assert east["classification"] == "data_only" and east["inputs"][0]["base"] is None + assert east["attribution"]["rows"] == [] and "无法读取" in east["attribution"]["note"] + + +def test_runs_without_metric_cards_still_compare_queries(): + base, target = Fake("r1"), Fake("r2") + base.tool("fetch", region_sql()) + target.tool("fetch", region_sql(WEEK37), rows=NEXT_ROWS) + result = run_compare(base, target) + assert result["metrics"] == [] and sum(result["summary"].values()) == 0 + assert any("没有口径卡" in n for n in result["notes"]) + assert [q["structure"] for q in result["queries"]] == ["params_only"] + + +def test_the_comparison_does_not_touch_the_runs(): + base, target = week("r1"), week("r2", NEXT_ROWS) + before = (list(base.events), list(target.events), dict(base.store), dict(target.store)) + run_compare(base, target) + assert (base.events, target.events, base.store, target.store) == before From c89aaba10ce53c8360dbe51d4bed5a63bbaf7933 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 08:52:04 +0800 Subject: [PATCH 2/8] =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=AF=B9=E6=AF=94?= =?UTF-8?q?=EF=BC=9AAgent=20=E6=8D=A2=E4=BA=86=E6=9F=A5=E8=AF=A2=E5=85=88?= =?UTF-8?q?=E5=90=8E=E6=97=B6=E6=8C=89=20SQL=20=E9=85=8D=E5=AF=B9=EF=BC=8C?= =?UTF-8?q?Agent=20=E6=94=B9=E4=BA=86=E9=85=8D=E7=BD=AE=E6=97=B6=E6=8C=89?= =?UTF-8?q?=E5=87=BA=E5=A4=84=E4=BA=A4=E6=9D=A5=E7=9A=84=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E6=8A=A5=E6=88=90=E6=95=B0=E6=8D=AE=E5=8F=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 查询配对改成一轮比一轮宽：同一节点、同一工具里规范化 SQL 相同的先配上，剩下的再按「第几次」配，最后跨节点按 SQL 配。以前严格按第几次配，Agent 这次换了查询的先后，两条都被配成「结构变了」，没按出处交的字段跟着被判成 SQL 变了。 Agent 按出处交来的字段：SQL 没变而 Agent 自己的配置（提示词、工具）变了时，它可能换了取哪一格，归为模板变化 （sql_changed）并写明是哪个 Agent；调用工具节点的格子由口径卡的 cell() 定，不受影响。 --- backend/app/engine/compare.py | 53 ++++++++++++++++++---------- backend/tests/test_compare_engine.py | 33 +++++++++++++++++ 2 files changed, 68 insertions(+), 18 deletions(-) diff --git a/backend/app/engine/compare.py b/backend/app/engine/compare.py index 32e022f0..da41a670 100644 --- a/backend/app/engine/compare.py +++ b/backend/app/engine/compare.py @@ -7,9 +7,10 @@ 怎么对： - 查询：两次运行各自封存范围内交回过的查询快照，按「节点 + 该节点里第几次查询 + 工具」配对，配不上的再按规范化 - SQL（data/sql_semantics.normalize_sql）配。结构：规范文本和字面量都相同是 same；规范文本相同、只有字面量不同是 - params_only（定时运行每次的日期不同，属于参数变化不是模板变化）；否则 changed（数据源换了也算）。核查类的查询 - （双盲复算、反方审稿发的 role: verify）不进报告，不参与对比。 + SQL（data/sql_semantics.normalize_sql）配；同一节点、同一工具里规范化 SQL 相同的先配上（Agent 每次查询的先后 + 可能不同，见 _pair）。结构：规范文本和字面量都相同是 same；规范文本相同、只有字面量不同是 params_only（定时运行 + 每次的日期不同，属于参数变化不是模板变化）；否则 changed（数据源换了也算）。核查类的查询（双盲复算、反方审稿发的 + role: verify）不进报告，不参与对比。 - 口径卡：同一个口径卡节点按指标 id 配对；循环里执行了几次的取最后一次。表达式、口径版本、小数位、单位、显示格式 有一样不同就是口径变了：小数位改变取整，单位、比率与否改变数值的含义，都不只是换个写法。 - 每个指标一个分类，按这个次序判：一边有一边没有（added / removed）；口径变了（caliber_and_data，可能连同数据）； @@ -214,22 +215,31 @@ def _queries(side: Side) -> list[_Query]: def _pair(base: list[_Query], target: list[_Query], text: Callable[[_Query], str | None]) -> list[tuple[_Query | None, _Query | None]]: - """先按「节点 + 第几次 + 工具」配，配不上的再按规范化 SQL 配（节点换了 ID、同一个模板只有日期不同的也配得上）。 + """配对，一轮比一轮宽： + 1. 同一节点、同一工具、规范化 SQL 相同（只有字面量不同也算）：Agent 每次查询的先后可能不同，按「第几次」硬配 + 会把换了先后的两条都配成「结构变了」；循环里同一个模板查几次的，按先后一一配上，和按第几次配一样 + 2. 剩下的按「节点 + 第几次 + 工具」配：同一个位置上的查询换了写法，就是这条查询的结构变了 + 3. 再剩下的跨节点按规范化 SQL 配：节点换了 ID，查的还是同一个模板 结果按这次运行的先后排，基准运行多出来的接在后面。""" - slots = {(q.node_id, q.ordinal, q.tool): i for i, q in enumerate(base)} matched: dict[int, int] = {} - for j, q in enumerate(target): - i = slots.get((q.node_id, q.ordinal, q.tool)) - if i is not None: - matched[j] = i - taken = set(matched.values()) - for j, q in enumerate(target): - if j in matched or (key := text(q)) is None: - continue - i = next((i for i, b in enumerate(base) if i not in taken and text(b) == key), None) - if i is not None: - matched[j] = i - taken.add(i) + taken: set[int] = set() + + def link(same: Callable[[_Query, _Query], bool]) -> None: + for j, q in enumerate(target): + if j in matched: + continue + i = next((i for i, b in enumerate(base) if i not in taken and same(b, q)), None) + if i is not None: + matched[j] = i + taken.add(i) + + def same_sql(b: _Query, q: _Query) -> bool: + key = text(q) + return key is not None and text(b) == key + + link(lambda b, q: b.node_id == q.node_id and b.tool == q.tool and same_sql(b, q)) + link(lambda b, q: (b.node_id, b.ordinal, b.tool) == (q.node_id, q.ordinal, q.tool)) + link(same_sql) pairs: list[tuple[_Query | None, _Query | None]] = [ (base[matched[j]] if j in matched else None, q) for j, q in enumerate(target)] return pairs + [(b, None) for i, b in enumerate(base) if i not in taken] @@ -654,7 +664,14 @@ class _Compare: qb, qt = self._side_query(self.base_queries, b), self._side_query(self.target_queries, t) if qb is None or qt is None: return _Change("unknown", f"输入「{path}」落在的查询快照不在封存范围内的查询里，无法比较取数") - return self._query_change(qb, qt) + change = self._query_change(qb, qt) + agent = t.item.get("node_id") + if t.item.get("via") == "agent_field" and agent and self.changed.get(str(agent)) \ + and change.kind in ("same", "params"): + # Agent 按出处交来的字段：SQL 没变，Agent 自己改了配置（提示词、工具……）也可能换了取哪一格。 + # 调用工具节点的格子由口径卡的 cell() 定，节点配置变了要么是 SQL 变了（上面已经比过），要么不影响取值 + return _Change("template", f"Agent「{self.title(agent)}」的配置有变化，它按出处交来的字段可能换了取法") + return change nb, nt = b.item.get("node_id"), t.item.get("node_id") if nb != nt: return _Change("template", f"输入「{path}」的来源节点由「{self.title(nb)}」换成了「{self.title(nt)}」") diff --git a/backend/tests/test_compare_engine.py b/backend/tests/test_compare_engine.py index 2b7a49c7..c80ca91e 100644 --- a/backend/tests/test_compare_engine.py +++ b/backend/tests/test_compare_engine.py @@ -497,6 +497,39 @@ def test_queries_fall_back_to_normalized_sql_pairing(): assert len(result["queries"]) == 2 +def test_an_agent_asking_in_another_order_is_paired_by_sql(): + """Agent 这次换了查询的先后：同一节点里先按规范化 SQL 配，不按第几次硬配成两条「结构变了」。""" + by_region = region_sql() + by_day = "SELECT day, SUM(amount) AS gmv FROM orders WHERE day >= '2026-08-31' GROUP BY day" + + def make(run_id, sqls): + run = Fake(run_id) + run.add("node.started", "bot", {"node_type": "agent"}) + for i, sql in enumerate(sqls): + artifact = run.snapshot(sql, ["k", "gmv"], [["甲", 1.0]]) + run.add("tool.start", "bot", {"tool": "db_query__shop", "args": {"sql": sql}, "call_id": f"c{i}"}) + run.add("tool.end", "bot", {"tool": "db_query__shop", "call_id": f"c{i}", "query_artifact": artifact}) + run.add("node.finished", "bot", {"duration_ms": 1, "attempt": 1}) + return run + + result = run_compare(make("r1", [by_region, by_day]), + make("r2", [by_day.replace("2026-08-31", "2026-09-07"), region_sql(WEEK37)])) + pairs = [(q["structure"], q["base_sql"][:20]) for q in result["queries"]] + assert pairs == [("params_only", by_day[:20]), ("params_only", by_region[:20])] + + +def test_an_agent_reconfigured_with_the_same_sql_is_a_template_change(): + """Agent 改了提示词、SQL 一字不差：它按出处交来的字段可能换了取法，不能报成数据变化。""" + graph = weekly_graph() + graph["nodes"][2]["data"]["config"]["prompt"] = "按区域列出金额，从大到小" + result = run_compare(week("r1"), week("r2", NEXT_ROWS, graph=graph)) + metrics = by_id(result) + assert metrics["total"]["classification"] == "sql_changed" + assert any("区域分析" in r for r in metrics["total"]["reasons"]) + # 取数节点的格子不经过这个 Agent：照旧是数据变化 + assert metrics["east"]["classification"] == "data_only" + + def test_unpaired_queries_are_added_or_removed(): base = week("r1") target = week("r2") From 467f9599fd3c8c8d57736b5b7cc4313817f2a6d7 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 08:55:57 +0800 Subject: [PATCH 3/8] =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=AF=B9=E6=AF=94?= =?UTF-8?q?=EF=BC=9A=E8=A1=A5=E5=90=88=E5=B9=B6=E6=9F=A5=E8=AF=A2=E3=80=81?= =?UTF-8?q?=E5=BE=AA=E7=8E=AF=E9=87=8C=E5=A4=9A=E6=AC=A1=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E7=9A=84=E9=85=8D=E5=AF=B9=E6=B5=8B=E8=AF=95=EF=BC=8C=E5=8F=A3?= =?UTF-8?q?=E5=BE=84=E5=8D=A1=E8=AF=BB=E4=B8=8D=E5=87=BA=E6=9D=A5=E6=97=B6?= =?UTF-8?q?=E7=9A=84=E8=AF=B4=E6=B3=95=E8=A1=A5=E4=B8=8A=E3=80=8C=E7=9A=84?= =?UTF-8?q?=E3=80=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 合并查询只记在节点台账里、SQL 另在 merge.end 里，快照文件不在了照样比得了 SQL；循环里同一个节点执行几次的按先后 一一配上，多出来的一次标「删除」。口径卡指标集读不出来、被改过时的说明改成「口径卡「X」的指标集与哈希不一致」。 --- backend/app/engine/compare.py | 5 ++-- backend/tests/test_compare_api.py | 1 - backend/tests/test_compare_engine.py | 42 ++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/backend/app/engine/compare.py b/backend/app/engine/compare.py index da41a670..f30dd816 100644 --- a/backend/app/engine/compare.py +++ b/backend/app/engine/compare.py @@ -317,11 +317,12 @@ def _cards(side: Side) -> dict[str, _Card]: def _card_problem(load: Callable[[str], Any], artifact: str) -> str: + """指标集为什么用不了，接在「口径卡「X」」后面。""" try: content = load(artifact) except ValueError: - return "指标集与哈希不一致，疑似被修改" - return "无法读取指标集" if content is None else "指标集的内容不完整" + return "的指标集与哈希不一致，疑似被修改" + return "的指标集无法读取" if content is None else "的指标集内容不完整" def _plan_card(side: Side, card: _Card, queries: list[_Query]) -> None: diff --git a/backend/tests/test_compare_api.py b/backend/tests/test_compare_api.py index 7e92273a..cd2cf0d9 100644 --- a/backend/tests/test_compare_api.py +++ b/backend/tests/test_compare_api.py @@ -28,7 +28,6 @@ from app.db.models import Artifact, DataSource, Run, RunEvent, Workflow from app.engine.runner import run_manager, verify_manifest from app.main import app from tests.fixtures.compare_runs import ( - BASE_ROWS, EAST, NEXT_ROWS, Fake, diff --git a/backend/tests/test_compare_engine.py b/backend/tests/test_compare_engine.py index c80ca91e..519fcba3 100644 --- a/backend/tests/test_compare_engine.py +++ b/backend/tests/test_compare_engine.py @@ -16,6 +16,7 @@ import json from decimal import Decimal from app.engine import compare +from app.engine.merge_query import MERGE_SOURCE from tests.fixtures.compare_runs import ( BASE_ROWS, EAST, @@ -530,6 +531,47 @@ def test_an_agent_reconfigured_with_the_same_sql_is_a_template_change(): assert metrics["east"]["classification"] == "data_only" +def test_a_merge_query_is_compared_by_its_merge_sql(): + """合并查询只记在节点的台账里（没有 tool.end），SQL 在快照和 merge.end 里；快照文件不在了也比得了。""" + def make(run_id, sql, *, drop=False): + run = Fake(run_id) + run.tool("fetch", region_sql()) + snap = {"columns": ["region", "gmv"], "rows": [["东区", 1.0]], "truncated": False, "sql": sql, + "source": MERGE_SOURCE, "sources": ["shop"]} + artifact = run.put(snap) + run.add("node.started", "join", {"node_type": "merge"}) + run.add("merge.end", "join", {"sql": sql, "query_artifact": artifact, "rows": 1}) + run.add("node.finished", "join", {"duration_ms": 1, "attempt": 1, "evidence": [ + {"kind": "query", "node_id": "join", "exec": 1, "artifact": artifact, "source": MERGE_SOURCE, + "columns": ["region", "gmv"], "rows": 1, "truncated": False}]}) + if drop: + del run.store[artifact] + return run + + result = run_compare(make("r1", "SELECT region, gmv FROM a WHERE gmv > 10"), + make("r2", "SELECT region, gmv FROM a WHERE gmv > 20")) + join = result_queries(result, "join") + assert (join["ordinal"], join["structure"], join["literals"]) == (1, "params_only", [{"base": "10", "target": "20"}]) + result = run_compare(make("r1", "SELECT region, gmv FROM a WHERE gmv > 10", drop=True), + make("r2", "SELECT region, gmv FROM a WHERE gmv > 10 AND region <> '西区'")) + join = result_queries(result, "join") + assert join["structure"] == "changed" and join["base_sql"] == "SELECT region, gmv FROM a WHERE gmv > 10" + assert "无法读取" in join["problem"] + + +def test_a_node_run_in_a_loop_is_paired_execution_by_execution(): + def make(run_id, regions): + run = Fake(run_id) + for region in regions: + run.tool("fetch", f"SELECT SUM(amount) AS gmv FROM orders WHERE region = '{region}'", ["gmv"], [[1.0]], + types={"gmv": "number"}) + return run + + result = run_compare(make("r1", ["东区", "南区", "西区"]), make("r2", ["东区", "北区"])) + assert [(q["ordinal"], q["structure"]) for q in result["queries"]] == [(1, "same"), (2, "params_only"), (3, "removed")] + assert result["queries"][1]["literals"] == [{"base": "'南区'", "target": "'北区'"}] + + def test_unpaired_queries_are_added_or_removed(): base = week("r1") target = week("r2") From 1e90a5a5022f87b35f04f0e4568c0904c88a13d9 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 08:58:18 +0800 Subject: [PATCH 4/8] =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=AF=B9=E6=AF=94?= =?UTF-8?q?=EF=BC=9A=E7=BB=93=E6=9E=9C=E9=87=8C=E4=B8=8D=E5=87=BA=E7=8E=B0?= =?UTF-8?q?=E9=9D=9E=E6=95=B0=E5=92=8C=E6=97=A0=E7=A9=B7=E5=A4=A7=EF=BC=8C?= =?UTF-8?q?=E7=BB=B4=E5=BA=A6=E5=88=97=E7=9A=84=E9=9D=9E=E6=95=B0=E5=BD=93?= =?UTF-8?q?=E6=96=87=E5=AD=97=E9=85=8D=E5=AF=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 接口按 allow_nan=False 写 JSON，结果里混进一个 NaN 或 Infinity 就是 500。指标值、输入值、十进制相减原本就只出 有限的数；剩下两处补上：维度列里的非数（float NaN）写成文字再当键，度量加起来、占比算出来超出浮点范围的给空。 --- backend/app/engine/compare.py | 16 +++++++++++----- backend/tests/test_compare_engine.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/backend/app/engine/compare.py b/backend/app/engine/compare.py index f30dd816..c6e88e93 100644 --- a/backend/app/engine/compare.py +++ b/backend/app/engine/compare.py @@ -33,6 +33,7 @@ from __future__ import annotations import ast +import math from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, field from decimal import Decimal, InvalidOperation @@ -355,11 +356,13 @@ def _exact(value: Any) -> Decimal | None: return None -def _plain(d: Decimal, *, whole: bool) -> int | float: - """十进制换回 JSON 里的数：参与的都是整数时给整数，否则给浮点。""" +def _plain(d: Decimal, *, whole: bool) -> int | float | None: + """十进制换回 JSON 里的数：参与的都是整数时给整数，否则给浮点。浮点放不下的（超过 1e308）给 None：接口按 JSON + 交回，写不出无穷大。""" if whole and d == d.to_integral_value(): return int(d) - return float(d) + value = float(d) + return value if math.isfinite(value) else None def _places(*metrics: dict[str, Any]) -> int | None: @@ -853,7 +856,7 @@ class _Compare: delta = _plain(t - b, whole=whole) meta = {**tm, "decimals": _places(bm, tm)} out.update(delta=delta, delta_rendered=_signed(meta, delta) if delta != 0 else None, - delta_pct=float(round((t - b) / abs(b) * 100, 2)) if b != 0 else None) + delta_pct=_plain(round((t - b) / abs(b) * 100, 2), whole=False) if b != 0 else None) def _decompose(self, bm: dict[str, Any], tm: dict[str, Any], b: _Card, t: _Card, bp: Any, tp: Any) -> tuple[dict[str, Any] | None, str | None]: @@ -992,7 +995,7 @@ class _Compare: rows.append((abs(delta), {"key": list(key), "base": None if vb is None else _plain(vb, whole=False), "target": None if vt is None else _plain(vt, whole=False), "delta": _plain(delta, whole=False), - "share": float(delta / total) if share and total != 0 else None})) + "share": _plain(delta / total, whole=False) if share and total != 0 else None})) rows.sort(key=lambda r: -r[0]) # 稳定排序：变化一样大的按先出现的 if len(rows) > TOP_KEYS: notes.append(MORE_KEYS.format(count=len(rows), top=TOP_KEYS)) @@ -1120,6 +1123,9 @@ def _dimensions(sb: dict[str, Any], st: dict[str, Any], rb: list[tuple[int, dict def _hashable(value: Any) -> Any: + """维度值当键：文本、数、布尔、空值原样；别的（列表、对象）和非数、无穷大写成文字——键要能比较，也要写得进 JSON。""" + if isinstance(value, float) and not math.isfinite(value): + return repr(value) return value if isinstance(value, (str, int, float, bool)) or value is None else repr(value) diff --git a/backend/tests/test_compare_engine.py b/backend/tests/test_compare_engine.py index 519fcba3..1d2d9f3f 100644 --- a/backend/tests/test_compare_engine.py +++ b/backend/tests/test_compare_engine.py @@ -316,6 +316,21 @@ def test_a_period_column_that_never_matches_is_not_a_dimension(): assert "day" in attribution["note"] +def test_odd_values_stay_json_safe(): + """维度列里有非数、度量加起来超出浮点范围：结果照样写得进 JSON（接口按 allow_nan=False 写，写不进就是 500）。""" + def make(run_id, rows): + run = Fake(run_id) + bot = run.agent("bot", region_sql(), ["region", "gmv"], rows, types={"region": "text", "gmv": "number"}) + # 卡上的值随便取一个有限的数（这里不重算它），两次不同就行 + run.card([metric("total", "sum(vars.kpi.amounts)", float(len(rows)), [amounts_in(bot, rows)])]) + return run + + nan = float("nan") + result = run_compare(make("r1", [["东区", 1.0], [nan, 2.0]]), make("r2", [["东区", 1e308], ["东区", 1e308], [nan, 1.0]])) + rows = {tuple(r["key"]): r for r in by_id(result)["total"]["attribution"]["rows"]} + assert rows[("东区",)]["target"] is None and rows[("nan",)]["delta"] == -1.0 + + # -------------------------------------------------------------------------- # 遮罩列 # -------------------------------------------------------------------------- From 63c54e1c204055682fa46d06eb145ea9d09cda44 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 09:00:09 +0800 Subject: [PATCH 5/8] =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=AF=B9=E6=AF=94?= =?UTF-8?q?=EF=BC=9Acompare()=20=E7=9A=84=E8=AF=B4=E6=98=8E=E9=87=8C?= =?UTF-8?q?=E5=86=99=E5=85=A8=E8=BF=94=E5=9B=9E=E7=9A=84=E5=BD=A2=E7=8A=B6?= =?UTF-8?q?=EF=BC=8C=E6=8E=A5=E5=8F=A3=E8=AF=B4=E6=98=8E=E6=8C=87=E8=BF=87?= =?UTF-8?q?=E6=9D=A5=E6=97=B6=E5=89=8D=E7=AB=AF=E7=85=A7=E7=9D=80=E5=B0=B1?= =?UTF-8?q?=E8=83=BD=E8=AF=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 除接口约定的字段外，几处补充也写明：模板节点的 labels（配置项的中文叫法）、输入的 masked 和两边的快照 id、查询的 两边快照 id、differences 和 problem；归不了因时 rows 为空、total_delta 为空、原因在 note。 --- backend/app/engine/compare.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/backend/app/engine/compare.py b/backend/app/engine/compare.py index c6e88e93..6fe89179 100644 --- a/backend/app/engine/compare.py +++ b/backend/app/engine/compare.py @@ -125,8 +125,25 @@ class Side: def compare(base: Side, target: Side) -> dict[str, Any]: - """对比两次运行（base 是基准，target 是这次）：{target, base, summary, template, metrics, queries, notes}， - 形状见接口约定（api/compare.py）。只读，不碰两次运行的任何东西。""" + """对比两次运行（base 是基准，target 是这次），只读，不碰两次运行的任何东西。返回： + + - target / base：两次运行的概要（Side.run 原样） + - summary：各类指标的个数，键是 CLASSES 里的八类 + - template：{base_version, target_version, same_graph, nodes: [{node_id, title, change: added | removed | config, + keys: [配置键名，值是字典的往下一层写 args.sql], labels: [同序的中文叫法]}]}。只列键名，不展示配置的值 + - metrics：每个指标一项 {card_id, card_title, metric_id, name, unit, decimals, base, target（有限的数，否则 + None）, base_rendered, target_rendered（卡上记的写法）, delta, delta_rendered（为 0、不能相减时为 None）, + delta_pct（相对基准的百分比，两位小数；基准为 0 时为 None）, classification, reasons: [说明], decomposition: + {caliber_effect, data_effect, caliber_rendered, data_rendered} 或 None, decomposition_note（为什么没拆）, + inputs: [{path, base, target, masked?, base_artifact?, target_artifact?}], attribution} + - attribution：{columns, measure, rows: [{key: [维度值…], base, target, delta, share}], total_delta, note} 或 + None。归不了因的（快照截断、度量列遮罩、快照读不出来、维度列每期都不同）rows 为空、total_delta 为 None，原因 + 写在 note；一边没有的维度值那一边写 None + - queries：每对查询一项 {node_id, node_title, ordinal, structure, base_sql, target_sql, literals: [{base, + target}]（只列不同的字面量）, base_rows, target_rows, base_truncated, target_truncated, base_artifact, + target_artifact, differences: [结构上差在哪], problem?（快照读不出来、被改过）} + - notes：整体的说明：连线、默认设置、运行输入的变化，口径卡执行了几次、读不出来，遮罩 + """ return _Compare(base, target).result() From d176ff77ff4ead6567042da0b169029b554b05cb Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 10:20:29 +0800 Subject: [PATCH 6/8] =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=AF=B9=E6=AF=94?= =?UTF-8?q?=E5=A4=8D=E6=A0=B8=E4=BF=AE=E6=AD=A3=EF=BC=88=E7=AC=AC=201=20?= =?UTF-8?q?=E8=BD=AE=EF=BC=89=EF=BC=9A=E8=BE=93=E5=85=A5=E7=9A=84=E5=87=BA?= =?UTF-8?q?=E5=A4=84=E8=BF=BD=E5=88=B0=E5=90=88=E5=B9=B6=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E7=9A=84=E4=B8=8A=E6=B8=B8=E3=80=81Agent=20=E5=BC=95=E7=94=A8?= =?UTF-8?q?=E7=9A=84=E5=88=97=E5=92=8C=E8=A1=8C=E6=AE=B5=EF=BC=8C=E6=B2=A1?= =?UTF-8?q?=E5=87=BA=E5=A4=84=E7=9A=84=E6=95=B0=E3=80=81=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=B2=A1=E5=8F=98=E6=95=B0=E5=8D=B4=E5=8F=98=E4=BA=86=E7=9A=84?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E6=8A=A5=E6=88=90=E6=95=B0=E6=8D=AE=E5=8F=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 出处判定（每个指标的分类靠它）： - 合并查询：读合并结果的指标按别名一路追到喂它的每一份上游快照（嵌套的合并也追），上游 SQL 结构变了是 sql_changed， 不拆、不归因；合并节点多接、少接了输入也算结构变化 - 图里写死的字面量改了（'paid' 改成 'refunded'、系数、LIMIT）：执行的 SQL 只差字面量，但图里的 SQL 改过，算模板变化； 图里一字未改（定时运行代入的日期）才是参数。图里的 SQL 只改了空白、大小写的不算改过 - Agent 按出处交来的字段：还要引用同一列、同一行（段），两边都引用整份结果的不算换了；换了列、换了行段的说不清 （incomparable），不再把没引用的区域算成「数据没了」去归因 - 模型、人直接交出的值（Agent 没按出处交、模型调用、人工审批……）：值变了说不清是数据还是模型本身，判 incomparable； 它自己查的库结构变了、配置变了照常标出 - 代码等确定性节点：上游的查询结果两次一模一样、交出的值却变了的，判 incomparable，不再说「变化来自上游数据」； 上游只有运行输入变了的算参数 - 图级变化落到节点上：连线有增删的终点节点、默认设置改了且没有自行设置这几项的模型类节点，都算改了配置； 「模板和取数的 SQL 都没有变化」只在整张图都没变时说 - 运行输入是列表、对象的，按还原出来的完整值比，长度不变、内容变了的是参数变化 - 同一节点按 SQL 配对时先配字面量也相同的：Agent 换了先后查东区、南区，两条各自配上自己，不再配成两条「只有参数不同」 - 一边记不出输入是哪个节点写的，说清是哪一边没记录（不再写出「None」） 口径与数值： - 口径变了的指标，喂它的取数也变了、说不清的照样写进说明（只在一边才有的输入是改表达式带来的，不重复说）； 分类次序写进 compare() 的说明 - 千分位和不分组互换只影响显示，写进说明、不算口径变化；比率与否不同才算 - 升级前的口径卡没记小数位、显示格式：按那次运行图里的指标定义核对，定义也没有就不比，不再误报「小数位由 未设置 改为 1」 - 百分比：基准极小、这次很大时十进制量化放不下，退回浮点算，不再整个对比报 500 - 没设小数位的指标拆分：三个数按同一精度量化（最多 15 位有效数字），转成浮点交出去后两效应之和仍逐位等于总变化 - 维度归因标出用的是哪个输入、两边各自哪份快照（path、base_artifact、target_artifact） 接口：已经结束、只是没有封存清单的运行，409 说「没有封存清单，不能用于对比」，不再说「运行结束后才能对比」。 测试：补上复核指出的必测项（只一边对得上不给占比、只截断基准一侧、换了数据源、按旧口径小数位取整、写它的节点多查一次、 口径和 SQL 一起变不归因、小数位和比率变化、日期类型和常量列不当维度、合计行浮点误差、只改名称、结束过又接着运行的不当 基准），以及上面每一处修正的用例。32 处变异逐条打红验证，全部被测试抓到。 --- backend/app/api/compare.py | 10 +- backend/app/engine/compare.py | 588 ++++++++++++++++++++----- backend/tests/fixtures/compare_runs.py | 62 ++- backend/tests/test_compare_api.py | 14 +- backend/tests/test_compare_engine.py | 435 +++++++++++++++++- 5 files changed, 978 insertions(+), 131 deletions(-) diff --git a/backend/app/api/compare.py b/backend/app/api/compare.py index 5b648c45..6d5a7b2b 100644 --- a/backend/app/api/compare.py +++ b/backend/app/api/compare.py @@ -7,8 +7,8 @@ GET /api/runs/{run_id}/compare/base：默认拿哪一次来比，{base_run_id, r GET /api/runs/{run_id}/compare?base={base_id}：{target, base, summary, template, metrics, queries, notes}，每个字段 的形状见 engine/compare.py。两次运行都必须已结束、封存且封存核对一致，封存判定照搬证据接口（api/evidence._sealed）： -没封存的运行事件还会变，封存核对不一致的事件被改过，拿它们比出来的结论都不作数，回 409。不是同一个工作流、或者 -是同一次运行，回 422。 +没封存的运行事件还会变，封存核对不一致的事件被改过，拿它们比出来的结论都不作数，回 409（还没结束的和已经结束却没有 +封存清单的分开说：后者再等也等不来封存）。不是同一个工作流、或者是同一次运行，回 422。 只读：两次运行都不追加事件、不落工件、不改运行记录；结果只交回给发请求的人。只认各自封存范围内的工件（证据接口的 白名单，加上封存的节点产出和指标集里写着的完整值，同推演），遮罩列（查询当时记下的加数据源现在设的）的值不出现在 @@ -48,6 +48,8 @@ SAME_RUN = "不能和同一次运行自己对比，请选择另一次运行作 UNSAVED = "{who}不属于任何已保存的工作流，无法确认两次运行是同一个工作流，不能对比" OTHER_WORKFLOW = "两次运行不属于同一个工作流（「{target}」与「{base}」），不能对比" UNSEALED_TEXT = "{who}尚未结束并封存：对比只用封存的运行记录，运行结束后才能对比" +#: 已经结束、只是没有封存清单（封存上线前结束的、结束时没封上的）：再等也等不来封存，不能说「运行结束后才能对比」 +NO_MANIFEST_TEXT = "{who}没有封存清单，不能用于对比" SEAL_BROKEN_TEXT = "{who}的封存核对不一致：封存后有事件被修改、删除或插入，不能用于对比。可在记录页点「核对」查看核对结果" NO_BASE_TEXT = ("没有找到可以对比的上一次运行：需要同一工作流、同一运行类别（定时任务发起的优先找同一定时任务）、" "更早结束并已封存的运行") @@ -120,8 +122,10 @@ async def compare_runs(run_id: str, base_id: str) -> dict[str, Any]: sealed = [] for run, who in ((target, TARGET), (base, BASE)): found = await _sealed(run.id) - if not found.sealed or found.status not in _TERMINAL: + if found.status not in _TERMINAL: raise CodedHTTPException(409, UNSEALED_TEXT.format(who=who), UNSEALED) + if not found.sealed: + raise CodedHTTPException(409, NO_MANIFEST_TEXT.format(who=who), UNSEALED) if not found.trusted: raise CodedHTTPException(409, SEAL_BROKEN_TEXT.format(who=who), SEAL_BROKEN) sealed.append(found) diff --git a/backend/app/engine/compare.py b/backend/app/engine/compare.py index 6fe89179..c899b533 100644 --- a/backend/app/engine/compare.py +++ b/backend/app/engine/compare.py @@ -7,22 +7,31 @@ 怎么对： - 查询：两次运行各自封存范围内交回过的查询快照，按「节点 + 该节点里第几次查询 + 工具」配对，配不上的再按规范化 - SQL（data/sql_semantics.normalize_sql）配；同一节点、同一工具里规范化 SQL 相同的先配上（Agent 每次查询的先后 - 可能不同，见 _pair）。结构：规范文本和字面量都相同是 same；规范文本相同、只有字面量不同是 params_only（定时运行 - 每次的日期不同，属于参数变化不是模板变化）；否则 changed（数据源换了也算）。核查类的查询（双盲复算、反方审稿发的 - role: verify）不进报告，不参与对比。 -- 口径卡：同一个口径卡节点按指标 id 配对；循环里执行了几次的取最后一次。表达式、口径版本、小数位、单位、显示格式 - 有一样不同就是口径变了：小数位改变取整，单位、比率与否改变数值的含义，都不只是换个写法。 -- 每个指标一个分类，按这个次序判：一边有一边没有（added / removed）；口径变了（caliber_and_data，可能连同数据）； - 喂它的查询结构变了、上游节点改了配置（sql_changed，标出、不拆）；输入对不上具体的查询，上游又有变化，说不清变化 - 从哪来（incomparable）；只有参数变了（params_only，含运行输入）；都没变时数值相等是 unchanged，不等是 data_only。 - 「喂它的查询」按输入的出处认：cell() 那一格、Agent 按出处交来的字段落在哪份快照上就是哪条查询（两边各自的快照 - 直接比 SQL）；不落在快照上的，看写它的节点自己查过的库，节点不查库（代码、数据整形……）就看它的配置和画布上的 - 全部上游。 + SQL（data/sql_semantics.normalize_sql）配；同一节点、同一工具里规范化 SQL 相同的先配上，字面量也相同的又在前 + （Agent 每次查询的先后可能不同，见 _pair）。结构：规范文本和字面量都相同是 same；规范文本相同、只有字面量不同是 + params_only（定时运行每次的日期不同，属于参数变化不是模板变化）；否则 changed（数据源换了也算）。核查类的查询 + （双盲复算、反方审稿发的 role: verify）不进报告，不参与对比。 +- 口径卡：同一个口径卡节点按指标 id 配对；循环里执行了几次的取最后一次。表达式、口径版本、小数位、单位、比率与否 + 有一样不同就是口径变了：小数位改变取整，单位、比率与否改变数值的含义，都不只是换个写法。千分位和不分组互换只影响 + 显示，写进说明、不算口径变化。升级前的卡没记小数位、显示格式的，按那次运行图里的指标定义核对，定义也没有就不比。 +- 每个指标一个分类，按这个次序判：一边有一边没有（added / removed）；口径变了（caliber_and_data，可能连同数据； + 喂它的取数也有变化的照样写进说明）；喂它的查询结构变了、上游节点改了配置（sql_changed，标出、不拆）；输入对不上、 + 说不清变化从哪来（incomparable）；只有参数变了（params_only，含运行输入）；都没变时数值相等是 unchanged， + 不等是 data_only。 +- 「喂它的查询」按输入的出处认： + - cell() 那一格、Agent 按出处交来的字段落在哪份快照上就是哪条查询，两边各自的快照直接比 SQL；快照是合并查询的 + 结果时，再按别名一路追到喂它的每一份上游快照。只差字面量时还要看图里的 SQL：图里的 SQL 改过（改了写死的筛选值、 + 系数）是模板变化，图里一字未改的才是运行时代入的参数 + - Agent 按出处交来的字段还要引用同一列、同一段行（两边都引用整份结果的不算换了）：取哪一列、哪几行是 Agent 每次 + 运行时定的，换了就不是同一个数 + - 不落在快照上的：模型、人交出的值（Agent 没按出处交、模型调用、人工审批……）没有出处，值变了说不清是数据还是它 + 本身的差异；代码、数据整形这类节点看它自己查过的库，不查库的看它的配置和画布上的全部上游，上游的查询结果两次 + 一模一样、值却变了的同样说不清 + - 图级的变化落到节点上：连线有增删的，终点节点算改了；默认设置改了的，没有自行设置这几项的模型类节点算改了 - 严格拆分（只对口径变了的）：旧表达式的每个叶子路径都能在这次的口径卡输入里取得值时， 口径效应 = f_new(x_new) − f_old(x_new)，数据效应 = f_old(x_new) − f_old(x_old)。f_old(x_new) 用推演重算 （engine/recompute.py）的还原和求值算，取整按旧口径卡的小数位；三个数按十进制精确相减，两效应之和逐位等于 - 总变化。取数的 SQL 也变了、用到了遮罩列、单位不同的不拆，说明为什么。 + 总变化（转成浮点后也逐位相加，见 _split_exact）。取数的 SQL 也变了、用到了遮罩列、单位不同的不拆，说明为什么。 - 维度归因：指标的输入落在查询快照的某一格（或 Agent 数组字段的一段行）、两边快照都没截断时，取快照里不是数值的 列作维度（遮罩的、日期类型的不算；两边都只有一个值的、两边没有一个值相同的——日期、期次这类每期都不同的列—— 也不拿来配对），按维度键配对两边的行，算每个键的 base / target / delta，按 |delta| 取前 10。只有度量列之和 @@ -44,9 +53,10 @@ from app.data.sql_semantics import NormalizedSql, diff_facets, facets, normalize from app.engine import recompute from app.engine.evidence import DEFAULT_FORMAT, RenderError, is_number, render_number from app.engine.expressions import ExpressionError, cell_value, leaf_refs, parse_expression -from app.engine.labels import field_label +from app.engine.labels import TYPE_LABEL, field_label from app.engine.merge_query import MERGE_SOURCE -from app.engine.nodes.metrics import _REF_HEAD +from app.engine.nodes.metrics import _REF_HEAD, _display_of +from app.engine.toolcalls import QUERY_PREFIX UNCHANGED = "unchanged" DATA_ONLY = "data_only" @@ -74,6 +84,18 @@ _MERGE_DIALECT = "sqlite" #: 日期类的列不当维度：每期都不同，按它配对两边的行一个都配不上 _TEMPORAL = frozenset({"date", "datetime", "time"}) _FORMAT_LABEL = {"thousands": "千分位", "plain": "不分组", "percent_of_ratio": "比率按百分数显示"} +_RATIO = "percent_of_ratio" +#: 图里写 SQL 的配置项：调用工具节点的参数、合并查询的合并 SQL +_SQL_KEY = {"tool": "args.sql", "merge": "sql"} +#: 交出的值由模型或人给出的节点（类型、口径卡记的来历）：同样的输入、同样的配置，交出的值也可能不同 +_GENERATED = frozenset({"agent", "llm", "supervisor", "human", "memory", "retrieve", "subgraph"}) +_GENERATED_VIA = frozenset({"agent", "agent_field", "llm", "team", "human", "memory", "retrieval", "subgraph"}) +#: 图级默认设置只有模型类节点在读（节点没配 provider、model 时回退到它，GraphSpec.defaults） +_MODEL_TYPES = frozenset({"llm", "agent", "supervisor"}) +#: 改了也不影响交出的值的默认设置：审批策略只管调用工具前问不问人 +_QUIET_DEFAULTS = frozenset({"approval"}) +#: 浮点能原样写回的十进制有效数字位数（DBL_DIG） +_FLOAT_DIGITS = 15 NO_CARDS = "两次运行都没有口径卡，没有可以对比的指标" EDGES_CHANGED = "节点之间的连线有变化" @@ -82,6 +104,8 @@ MASKED_NOTE = "部分列在数据源中设置了遮罩，对比中不显示这 NO_LINEAGE = "口径卡没有记录输入（生成于系统升级前），无法判断变化来源" DATA_FROM_QUERIES = "取数的 SQL 没有变化，查询结果有变化" DATA_FROM_UPSTREAM = "模板和取数的 SQL 都没有变化，变化来自上游数据" +#: 图有别的改动、都不在它的上游：不能说「模板没有变化」 +DATA_FROM_UPSTREAM_SCOPED = "它上游的节点和取数的 SQL 都没有变化，变化来自上游数据" SQL_NOT_SPLIT = "取数的 SQL 或上游配置有变化，变化无法在口径与数据之间拆分" NOT_NUMBERS = "有一边的值不是数值，无法拆分" SPLIT_MASKED = "用到了设置了遮罩的列，不做拆分" @@ -136,9 +160,13 @@ def compare(base: Side, target: Side) -> dict[str, Any]: delta_pct（相对基准的百分比，两位小数；基准为 0 时为 None）, classification, reasons: [说明], decomposition: {caliber_effect, data_effect, caliber_rendered, data_rendered} 或 None, decomposition_note（为什么没拆）, inputs: [{path, base, target, masked?, base_artifact?, target_artifact?}], attribution} - - attribution：{columns, measure, rows: [{key: [维度值…], base, target, delta, share}], total_delta, note} 或 - None。归不了因的（快照截断、度量列遮罩、快照读不出来、维度列每期都不同）rows 为空、total_delta 为 None，原因 - 写在 note；一边没有的维度值那一边写 None + - classification 按这个次序取第一个对得上的：added / removed → caliber_and_data → sql_changed → incomparable + → params_only → unchanged / data_only。口径变了的指标喂它的取数也变了时，分类仍是 caliber_and_data，取数的 + 变化接在口径的说明后面写进 reasons + - attribution：{path, base_artifact, target_artifact（归因用的是哪个输入、两边各自哪份快照）, columns, measure, + rows: [{key: [维度值…], base, target, delta, share}], total_delta, note} 或 None。归不了因的（快照截断、 + 度量列遮罩、快照读不出来、维度列每期都不同）rows 为空、total_delta 为 None，原因写在 note；一边没有的维度值 + 那一边写 None - queries：每对查询一项 {node_id, node_title, ordinal, structure, base_sql, target_sql, literals: [{base, target}]（只列不同的字面量）, base_rows, target_rows, base_truncated, target_truncated, base_artifact, target_artifact, differences: [结构上差在哪], problem?（快照读不出来、被改过）} @@ -168,6 +196,8 @@ class _Query: #: 快照读不出来的原因：missing / tampered problem: str | None = None source: str | None = None + #: 合并查询的各个输入 [{alias, node_id, artifact, …}]：快照里记的，快照读不出来时取 merge.end 的 + inputs: list[dict[str, Any]] | None = None def _snapshot(load: Callable[[str], Any], artifact: str) -> tuple[dict[str, Any] | None, str | None]: @@ -189,16 +219,20 @@ def _queries(side: Side) -> list[_Query]: seen: set[tuple[str, str]] = set() asked: dict[tuple[str, Any], str] = {} # (节点, call_id) → tool.start 里的 SQL merged: dict[str, str] = {} # 合并节点 → merge.end 里的 SQL + merged_inputs: dict[str, Any] = {} # 合并节点 → merge.end 里的输入 count: dict[str, int] = {} def add(node_id: str, artifact: str, tool: Any, call_id: Any, sql: Any, source: Any) -> None: count[node_id] = count.get(node_id, 0) + 1 snap, problem = _snapshot(side.load, artifact) own = snap.get("sql") if snap is not None and isinstance(snap.get("sql"), str) else None + source = (snap or {}).get("source") or source + inputs = (snap or {}).get("inputs") if snap is not None else merged_inputs.get(node_id) out.append(_Query(artifact=artifact, node_id=node_id, tool=str(tool) if tool else None, ordinal=count[node_id], call_id=str(call_id) if call_id else None, sql=own or (sql if isinstance(sql, str) and sql.strip() else None), snap=snap, - problem=problem, source=(snap or {}).get("source") or source)) + problem=problem, source=source, + inputs=inputs if source == MERGE_SOURCE and isinstance(inputs, list) else None)) for _, etype, node_id, data in side.events: if not node_id or not isinstance(data, dict) or data.get("role") == "verify": @@ -212,8 +246,10 @@ def _queries(side: Side) -> list[_Query]: seen.add((node_id, artifact)) add(node_id, artifact, data.get("tool"), data.get("call_id"), asked.get((node_id, data.get("call_id"))), None) - elif etype == EventType.MERGE_END and isinstance(data.get("sql"), str): - merged[node_id] = data["sql"] + elif etype == EventType.MERGE_END: + if isinstance(data.get("sql"), str): + merged[node_id] = data["sql"] + merged_inputs[node_id] = data.get("inputs") elif etype == EventType.NODE_FINISHED: for entry in data.get("evidence") or []: if not (isinstance(entry, dict) and entry.get("kind") == "query" and entry.get("artifact")): @@ -232,12 +268,13 @@ def _queries(side: Side) -> list[_Query]: def _pair(base: list[_Query], target: list[_Query], - text: Callable[[_Query], str | None]) -> list[tuple[_Query | None, _Query | None]]: + norm: Callable[[_Query], NormalizedSql | None]) -> list[tuple[_Query | None, _Query | None]]: """配对，一轮比一轮宽： - 1. 同一节点、同一工具、规范化 SQL 相同（只有字面量不同也算）：Agent 每次查询的先后可能不同，按「第几次」硬配 - 会把换了先后的两条都配成「结构变了」；循环里同一个模板查几次的，按先后一一配上，和按第几次配一样 - 2. 剩下的按「节点 + 第几次 + 工具」配：同一个位置上的查询换了写法，就是这条查询的结构变了 - 3. 再剩下的跨节点按规范化 SQL 配：节点换了 ID，查的还是同一个模板 + 1. 同一节点、同一工具、规范化 SQL 和字面量都相同：Agent 每次查询的先后可能不同，按「第几次」硬配会把换了先后的 + 两条都配成「结构变了」；先配一字不差的，换了先后的东区、南区两条才各自配上自己，不会配成两条「参数不同」 + 2. 同一节点、同一工具、规范化 SQL 相同（只有字面量不同）：循环里同一个模板查几次的，按先后一一配上 + 3. 剩下的按「节点 + 第几次 + 工具」配：同一个位置上的查询换了写法，就是这条查询的结构变了 + 4. 再剩下的跨节点按规范化 SQL 配（同样先配字面量也相同的）：节点换了 ID，查的还是同一个模板 结果按这次运行的先后排，基准运行多出来的接在后面。""" matched: dict[int, int] = {} taken: set[int] = set() @@ -251,12 +288,20 @@ def _pair(base: list[_Query], target: list[_Query], matched[j] = i taken.add(i) - def same_sql(b: _Query, q: _Query) -> bool: - key = text(q) - return key is not None and text(b) == key + def same_sql(b: _Query, q: _Query, *, literals: bool = False) -> bool: + nq = norm(q) + if nq is None: + return False + nb = norm(b) + return nb is not None and nb.text == nq.text and (not literals or nb.literals == nq.literals) - link(lambda b, q: b.node_id == q.node_id and b.tool == q.tool and same_sql(b, q)) + def here(b: _Query, q: _Query) -> bool: + return b.node_id == q.node_id and b.tool == q.tool + + link(lambda b, q: here(b, q) and same_sql(b, q, literals=True)) + link(lambda b, q: here(b, q) and same_sql(b, q)) link(lambda b, q: (b.node_id, b.ordinal, b.tool) == (q.node_id, q.ordinal, q.tool)) + link(lambda b, q: same_sql(b, q, literals=True)) link(same_sql) pairs: list[tuple[_Query | None, _Query | None]] = [ (base[matched[j]] if j in matched else None, q) for j, q in enumerate(target)] @@ -382,6 +427,44 @@ def _plain(d: Decimal, *, whole: bool) -> int | float | None: return value if math.isfinite(value) else None +def _percent(base: Decimal, target: Decimal) -> float | None: + """相对基准的百分比，两位小数；基准为 0 没有百分比。基准极小（浮点误差那么小的数）、这次很大时，百分比的整数 + 部分比十进制的默认精度（28 位）还长，量化到两位小数会抛 InvalidOperation：退回浮点算，浮点也放不下的给 None。""" + if base == 0: + return None + ratio = (target - base) / abs(base) * 100 + try: + return _plain(ratio.quantize(Decimal("0.01")), whole=False) + except ArithmeticError: + value = float(ratio) + return round(value, 2) if math.isfinite(value) else None + + +def _round_trips(d: Decimal) -> bool: + """这个十进制数换成浮点、再按最短写法写回来还是它自己（JSON 里交出去的就是浮点）。""" + value = float(d) + return math.isfinite(value) and Decimal(repr(value)) == d + + +def _split_exact(delta: Decimal, caliber: Decimal) -> tuple[Decimal, Decimal, Decimal]: + """(总变化, 口径效应, 数据效应)，三个数转成浮点交出去以后仍逐位相加。 + + 十进制里数据效应 = 总变化 − 口径效应是精确的，可转成浮点时三个数各舍入各的：没设小数位的指标（0.1 + 0.2 = + 0.30000000000000004 这种）交出去的两效应之和会和总变化差在最后一位。这时三个数按同一个精度量化（最多 15 位 + 有效数字，浮点能原样写回的位数）：总变化和口径效应各自量化，数据效应取两者之差。设了小数位的值本来就写得回， + 原样交回。""" + data = delta - caliber + if all(_round_trips(x) for x in (delta, caliber, data)): + return delta, caliber, data + top = max(abs(delta), abs(caliber), abs(data)) + for digits in range(_FLOAT_DIGITS, _FLOAT_DIGITS - 3, -1): + exp = Decimal(1).scaleb(top.adjusted() + 1 - digits) + d, c = delta.quantize(exp), caliber.quantize(exp) + if all(_round_trips(x) for x in (d, c, d - c)): + return d, c, d - c + return delta, caliber, data + + def _places(*metrics: dict[str, Any]) -> int | None: """几个指标里最多的小数位；有一个没设的就不定小数位（按值本来的样子写）。""" found = [m.get("decimals") for m in metrics] @@ -483,6 +566,38 @@ def _key_label(key: str, node_type: Any) -> str: return "节点类型" if key == "type" else field_label(key, node_type) +def _unset(value: Any) -> bool: + """节点上没写这一项：运行时回退到图级默认设置（NodeContext.cfg 同一个规则）。""" + return value in (None, "") + + +def _graph_sql(node: dict[str, Any] | None) -> tuple[str, str | None] | None: + """节点在图里写的 SQL 和它的数据源：调用工具节点是参数里的 SQL（数据源从工具名来），合并查询是合并 SQL。 + 别的节点（Agent 自己写 SQL）返回 None。""" + if node is None: + return None + config = _config(node) + if node.get("type") == "tool": + args = config.get("args") + sql = args.get("sql") if isinstance(args, dict) else None + tool = str(config.get("tool") or "") + source = tool[len(QUERY_PREFIX):] if tool.startswith(QUERY_PREFIX) else None + return (sql, source) if isinstance(sql, str) and sql.strip() else None + if node.get("type") == "merge": + sql = config.get("sql") + return (sql, MERGE_SOURCE) if isinstance(sql, str) and sql.strip() else None + return None + + +def _definition(graph: dict[str, Any], card_id: str, metric_id: str) -> dict[str, Any] | None: + """图里口径卡节点上写的指标定义；指标定义来自别的工作流（口径卡来源）的不算，那里写的才是当时用的。""" + card = _nodes(graph).get(card_id) + if card is None or card.get("type") != "metrics" or _config(card).get("caliber_from") not in (None, "", {}): + return None + return next((d for d in _config(card).get("metrics") or [] + if isinstance(d, dict) and str(d.get("id")) == metric_id), None) + + # -------------------------------------------------------------------------- # 输入的出处有没有变 # -------------------------------------------------------------------------- @@ -523,11 +638,12 @@ class _Compare: self._norm: dict[tuple[str, str | None], NormalizedSql] = {} self._shapes: dict[tuple[int, int], _Shape] = {} self._ancestors: dict[str, set[str]] = {} + self.base_nodes, self.target_nodes = _nodes(base.graph), _nodes(target.graph) self.base_queries, self.target_queries = _queries(base), _queries(target) - self.pairs = _pair(self.base_queries, self.target_queries, self._text) + self.pairs = _pair(self.base_queries, self.target_queries, self._norm_of) self.template, self.changed = self._template() - self.titles = {**{k: self._title_of(n) for k, n in _nodes(base.graph).items()}, - **{k: self._title_of(n) for k, n in _nodes(target.graph).items()}} + self.titles = {**{k: self._title_of(n) for k, n in self.base_nodes.items()}, + **{k: self._title_of(n) for k, n in self.target_nodes.items()}} self.parents: dict[str, set[str]] = {} for source, target_id, _ in _edges(base.graph) | _edges(target.graph): self.parents.setdefault(target_id, set()).add(source) @@ -576,8 +692,15 @@ class _Compare: self._norm[key] = normalize_sql(sql, dialect) return self._norm[key] - def _text(self, q: _Query) -> str | None: - return self._normalize(q.sql, self._dialect(q.source)).text if q.sql else None + def _norm_of(self, q: _Query) -> NormalizedSql | None: + return self._normalize(q.sql, self._dialect(q.source)) if q.sql else None + + def _same_graph_sql(self, a: tuple[str, str | None] | None, b: tuple[str, str | None] | None) -> bool: + """图里的两段 SQL 规范化以后一模一样（只差空白、大小写、注释）。""" + if a is None or b is None: + return False + na, nb = self._normalize(a[0], self._dialect(a[1])), self._normalize(b[0], self._dialect(b[1])) + return (na.text, na.literals) == (nb.text, nb.literals) def shape(self, b: _Query | None, t: _Query | None) -> _Shape: if b is None: @@ -656,10 +779,71 @@ class _Compare: return _Change("params", f"{name}的 SQL 只有参数不同（{shown}{more}）") return _Change("same") + def _snapshot_change(self, b: _Query | None, t: _Query | None, seen: frozenset[str] = frozenset()) -> _Change: + """一对查询的取数变没变：SQL 和数据源（_query_change）；只差字面量时再看图里的 SQL 改没改过；合并查询再 + 一路追到喂它的每一份上游快照。""" + change = self._query_change(b, t) + if b is None or t is None: + return change + found = [change] + if change.kind == "params" and (edited := self._edited_sql(b, t)) is not None: + found.append(edited) + if b.source == MERGE_SOURCE and t.source == MERGE_SOURCE: + found += self._merge_change(b, t, seen | {b.artifact, t.artifact}) + return _worst(found) + + def _edited_sql(self, b: _Query, t: _Query) -> _Change | None: + """执行的 SQL 只差字面量时，图里写的 SQL 改没改过。定时运行代进去的日期不同，图里一字未改：是参数。图里写死的 + 筛选值、系数改了（'paid' 改成 'refunded'、0.9 改成 0.8）：执行的 SQL 同样只差字面量，却是改了模板。规范化 + 以后一模一样的（只改了空白、大小写）不算改过。Agent 自己写的 SQL 图里没有，返回 None。""" + was, now = _graph_sql(self.base_nodes.get(b.node_id)), _graph_sql(self.target_nodes.get(t.node_id)) + if was is None or now is None or self._same_graph_sql(was, now): + return None + nb, nt = self._normalize(was[0], self._dialect(was[1])), self._normalize(now[0], self._dialect(now[1])) + shown = [f"{x} → {y}" for x, y in zip(nb.literals, nt.literals) if x != y] if nb.text == nt.text else [] + detail = "（" + "，".join(shown[:2]) + (f" 等 {len(shown)} 处" if len(shown) > 2 else "") + "）" if shown else "" + return _Change("sql", f"工作流里「{self.title(t.node_id)}」的 SQL 改过{detail}，不是运行时代入的参数") + + def _merge_change(self, b: _Query, t: _Query, seen: frozenset[str]) -> list[_Change]: + """合并查询的每个输入（按别名配对两边）各自的取数变没变。合并 SQL 一字未改，喂它的取数 SQL 加了筛选，合并 + 结果说的也已经不是同一回事。输入是另一个合并查询的结果时接着往上追。""" + name = f"合并查询「{self.title(t.node_id)}」" + before, after = _merge_inputs(b), _merge_inputs(t) + if before is None or after is None: + return [_Change("unknown", f"{name}没有记录它的输入，无法比较上游的取数")] + out = [] + for alias in dict.fromkeys([*after, *before]): + was, now = before.get(alias), after.get(alias) + if was is None or now is None: + out.append(_Change("sql", f"{name}的输入有变化：这次{'多了' if was is None else '少了'}「{alias}」")) + continue + if was in seen or now in seen: + continue # 合并查询的输入只能是上游的快照，成不了环；防一手 + qb = next((q for q in self.base_queries if q.artifact == was), None) + qt = next((q for q in self.target_queries if q.artifact == now), None) + if qb is None or qt is None: + out.append(_Change("unknown", f"{name}的输入「{alias}」落在的查询快照不在封存范围内，无法比较上游的取数")) + continue + change = self._snapshot_change(qb, qt, seen) + out.append(_Change(change.kind, f"{name}的输入「{alias}」：{change.reason}") if change.reason else change) + return out + def _node_change(self, node_id: str) -> _Change | None: """一个节点自己查过的库在两次运行之间变没变；它两边都没查过库返回 None。""" own = [(b, t) for b, t in self.pairs if (b and b.node_id == node_id) or (t and t.node_id == node_id)] - return _worst(self._query_change(b, t) for b, t in own) if own else None + return _worst(self._snapshot_change(b, t) for b, t in own) if own else None + + def _data_moved(self, scope: set[str]) -> bool: + """这几个节点查到的数据两次有没有不同：有一对查询的行（或列）不同就算有；快照读不出来、只在一边有的核对不了， + 也算有（不能说它们一样）。一条查询都没有的算没有。""" + for b, t in self.pairs: + if (t or b).node_id not in scope: + continue + if b is None or t is None or b.snap is None or t.snap is None: + return True + if b.snap.get("rows") != t.snap.get("rows") or b.snap.get("columns") != t.snap.get("columns"): + return True + return False def _upstream(self, node_id: str) -> set[str]: """画布上 node_id 的全部上游（两次运行的图合起来，宁可多算）。""" @@ -674,56 +858,137 @@ class _Compare: self._ancestors[node_id] = seen return self._ancestors[node_id] - def provenance(self, path: str, b: Any, t: Any) -> _Change: - """一个输入的出处在两次运行之间变没变（b、t 是推演重算还原出来的输入）。""" + def _type_of(self, node_id: str) -> str | None: + node = self.target_nodes.get(node_id) or self.base_nodes.get(node_id) + return str(node.get("type")) if node is not None else None + + def _moved(self, node_id: str) -> str: + """一个影响取值的节点变化（self.changed）写成一句话。""" + name = f"上游节点「{self.title(node_id)}」" + return {"added": f"{name}是这次新增的", "removed": f"{name}这次被删除了", "edges": f"{name}的连线有变化", + "defaults": f"工作流默认设置有变化，{name}没有单独设置这些项，随之改变" + }.get(self.changed.get(node_id, ""), f"{name}的配置有变化") + + def provenance(self, path: str, b: Any, t: Any, *, moved: bool = True) -> _Change: + """一个输入的出处在两次运行之间变没变（b、t 是推演重算还原出来的输入）。moved 是这个指标的值变没变：输入是 + 容器、完整值又还原不出来，核对不了它变没变时，按它推断。""" if b is None or t is None: return _Change("unknown", f"输入「{path}」只在一边的口径卡里有记录") if b.snapshot or t.snapshot: - # 落在查询快照上：两边各自的那份快照直接比 SQL，不管全局配对把它们配给了谁 - if not (b.snapshot and t.snapshot): - return _Change("unknown", f"输入「{path}」只有一边对得上查询快照，无法比较取数") - qb, qt = self._side_query(self.base_queries, b), self._side_query(self.target_queries, t) - if qb is None or qt is None: - return _Change("unknown", f"输入「{path}」落在的查询快照不在封存范围内的查询里，无法比较取数") - change = self._query_change(qb, qt) - agent = t.item.get("node_id") - if t.item.get("via") == "agent_field" and agent and self.changed.get(str(agent)) \ - and change.kind in ("same", "params"): - # Agent 按出处交来的字段：SQL 没变，Agent 自己改了配置（提示词、工具……）也可能换了取哪一格。 - # 调用工具节点的格子由口径卡的 cell() 定，节点配置变了要么是 SQL 变了（上面已经比过），要么不影响取值 - return _Change("template", f"Agent「{self.title(agent)}」的配置有变化，它按出处交来的字段可能换了取法") - return change + return self._linked(path, b, t) nb, nt = b.item.get("node_id"), t.item.get("node_id") if nb != nt: + if not nb or not nt: + side = "基准运行" if not nb else "这次运行" + return _Change("unknown", f"{side}没有记录输入「{path}」由哪个节点写入，无法比较来源") return _Change("template", f"输入「{path}」的来源节点由「{self.title(nb)}」换成了「{self.title(nt)}」") + same = _same_input(b, t) if _from_input(path, b.item): - if recompute._same(b.item.get("value"), t.item.get("value")): - return _Change("same") - return _Change("params", f"运行输入「{path}」由「{_shown(b.item.get('value'))}」" - f"改为「{_shown(t.item.get('value'))}」") - if nb and (own := self._node_change(str(nb))) is not None: - # 写它的节点自己查过库（Agent 没按出处交、调用工具节点的结果整份用）：它查的那几条就是喂它的查询 - if self.changed.get(str(nb)) and own.kind in ("same", "params"): - return _Change("template", f"上游节点「{self.title(nb)}」的配置有变化") + return self._run_input(path, b, t, same, moved=moved) + if same is None: + same = not moved + writer = str(nb) if nb else None + if writer and (self._type_of(writer) in _GENERATED + or {b.item.get("via"), t.item.get("via")} & _GENERATED_VIA): + return self._generated(path, writer, same) + own = self._node_change(writer) if writer else None + if writer and own is not None: + # 写它的节点自己查过库（调用工具节点的结果整份用、合并查询……）：它查的那几条就是喂它的查询 + if writer in self.changed and own.kind in ("same", "params"): + return _Change("template", self._moved(writer)) + if own.kind == "same" and not same and not self._data_moved({writer}): + return _Change("unknown", f"输入「{path}」来自「{self.title(writer)}」，它查到的数据两次相同，" + "交出的值却变了，无法判断变化来源") return own - if nb and self.changed.get(str(nb)): - return _Change("template", f"上游节点「{self.title(nb)}」的配置有变化") - # 写它的节点不查库（代码、数据整形、模型调用……）：看画布上它的全部上游；写它的节点不知道就看整张图 - scope = self._upstream(str(nb)) if nb else set(self.titles) - moved = [n for n in scope if self.changed.get(n)] - changes = [self._query_change(b2, t2) for b2, t2 in self.pairs if (t2 or b2).node_id in scope] - worst = _worst(changes) - where = f"来自「{self.title(nb)}」" if nb else "不知道由哪个节点写入" - if moved or worst.kind in ("sql", "template", "unknown"): + if writer and writer in self.changed: + return _Change("template", self._moved(writer)) + # 写它的节点不查库（代码、数据整形……）：看画布上它的全部上游；写它的节点不知道就看整张图 + scope = self._upstream(writer) if writer else set(self.titles) + moved_nodes = [n for n in scope if n in self.changed] + worst = _worst(self._query_change(b2, t2) for b2, t2 in self.pairs if (t2 or b2).node_id in scope) + where = f"来自「{self.title(writer)}」" if writer else "不知道由哪个节点写入" + if moved_nodes or worst.kind in ("sql", "template", "unknown"): return _Change("unknown", f"输入「{path}」{where}，对不上具体的查询；它的上游有变化，无法判断变化来源") + if worst.kind == "same" and not same and not self._data_moved(scope): + # 上游查到的数据两次一模一样，它交的值却变了：不是数据变的。上游只有运行输入变了的，算参数 + if (self.base.inputs or {}) != (self.target.inputs or {}) \ + and any(self._type_of(n) == "input" for n in scope): + return _Change("params", f"输入「{path}」{where}，它的上游只有运行输入有变化") + return _Change("unknown", f"输入「{path}」{where}，它上游的查询结果两次相同，交出的值却变了，" + "无法判断变化来源") return worst + def _linked(self, path: str, b: Any, t: Any) -> _Change: + """落在查询快照上的输入：两边各自的那份快照直接比取数，不管全局配对把它们配给了谁。Agent 按出处交来的字段还要 + 引用同一列、同一段行。""" + if not (b.snapshot and t.snapshot): + return _Change("unknown", f"输入「{path}」只有一边对得上查询快照，无法比较取数") + qb, qt = self._side_query(self.base_queries, b), self._side_query(self.target_queries, t) + if qb is None or qt is None: + return _Change("unknown", f"输入「{path}」落在的查询快照不在封存范围内的查询里，无法比较取数") + change = self._snapshot_change(qb, qt) + if "agent_field" not in (b.item.get("via"), t.item.get("via")): + # cell() 取哪一格由口径卡的表达式定（行列用到的变量是另外的输入，各比各的） + return change + agent = t.item.get("node_id") or b.item.get("node_id") + if (cited := self._citation(agent, b, t, qb, qt)) is not None: + change = _worst([change, cited]) + if agent and str(agent) in self.changed and change.kind in ("same", "params"): + # SQL 没变、引用的也是同一处，Agent 自己的配置（提示词、工具、模型……）变了：字段的意思可能已经不同 + return _Change("template", f"{self._moved(str(agent))}，它按出处交来的字段可能换了取法") + return change + + def _citation(self, agent: Any, b: Any, t: Any, qb: _Query, qt: _Query) -> _Change | None: + """Agent 按出处交来的字段两次引用的是不是同一处：同一列（对象数组是同一组列）、同一行或同一段行。行段不同、但 + 两边引用的都是整份结果（这次多了几行）的算同一处。不是同一处返回说明。""" + name = f"Agent「{self.title(agent)}」" if agent else "Agent" + lb, lt = _locator(b), _locator(t) + cb, ct = lb.get("column") or lb.get("columns"), lt.get("column") or lt.get("columns") + if cb != ct: + return _Change("unknown", f"{name}引用的列由{_columns_text(cb)}改为{_columns_text(ct)}，取的不是同一个数") + if (b.row, b.rows) == (t.row, t.rows) or (b.row is None and t.row is None and _whole(b, qb) and _whole(t, qt)): + return None + return _Change("unknown", f"{name}引用的行由{_rows_text(b)}改为{_rows_text(t)}，取的不是同一组数") + + def _generated(self, path: str, writer: str, same: bool) -> _Change: + """模型、人交出的值（Agent 没按出处交、模型调用、人工审批……）：没有出处，值变了说不清是数据变了还是它本身的 + 差异——同样的数据、同样的配置，模型两次交出的数也可能不同。它自己查的库结构变了、它的配置变了照常标出。""" + own = self._node_change(writer) + if own is not None and own.kind == "sql": + return own + if writer in self.changed: + return _Change("template", self._moved(writer)) + if own is not None and own.kind == "unknown": + return own + if same: + return own or _Change("same") + kind = TYPE_LABEL.get(self._type_of(writer) or "", "节点") + return _Change("unknown", f"输入「{path}」由节点「{self.title(writer)}」（{kind}）直接给出，没有对应的查询快照，" + "无法判断变化是否来自数据") + + def _run_input(self, path: str, b: Any, t: Any, same: bool | None, *, moved: bool) -> _Change: + """运行输入（input.*、入口节点写进 vars 的）：变了就是参数变化。列表、对象按还原出来的完整值比，卡上只记了 + 「[2 项]」这样的大概，长度不变、内容变了的不能当没变。""" + if same is None: + if moved: + return _Change("unknown", f"运行输入「{path}」是列表或对象，当时没有保存完整值，无法核对是否有变化") + return _Change("same") + if same: + return _Change("same") + if b.shape == "container" or t.shape == "container": + return _Change("params", f"运行输入「{path}」的内容有变化") + return _Change("params", f"运行输入「{path}」由「{_shown(b.item.get('value'))}」" + f"改为「{_shown(t.item.get('value'))}」") + # ---------------------------------------------------------------- 模板 def _template(self) -> tuple[dict[str, Any], dict[str, str]]: - """两次运行的图快照比较：新增、删除的节点，配置有变化的节点和配置项名（不展示值）。changed 是影响取值的 - 变化（新增、删除、类型或配置变了；只改名称的不算），判断输入的出处时用。""" - before, after = _nodes(self.base.graph), _nodes(self.target.graph) + """两次运行的图快照比较：新增、删除的节点，配置有变化的节点和配置项名（不展示值）。 + + changed 是影响取值的变化，判断输入的出处时用：节点 → added / removed / config / edges / defaults。只改名称的 + 不算；图里的 SQL 只改了空白、大小写的不算（规范化以后一模一样）。图级的变化也落到节点上：连线有增删的，终点 + 节点交出的值可能变了（接进来的东西、执行的条件变了）；默认设置改了的，没有自行设置这几项的模型类节点跟着变。""" + before, after = self.base_nodes, self.target_nodes nodes: list[dict[str, Any]] = [] changed: dict[str, str] = {} @@ -737,17 +1002,32 @@ class _Compare: nodes.append(entry(node_id, node, "added", [])) changed[node_id] = "added" continue - keys, renamed_only = _config_keys(was, node) + keys, _ = _config_keys(was, node) if keys: nodes.append(entry(node_id, node, "config", keys)) - if not renamed_only: + sql_key = _SQL_KEY.get(str(node.get("type"))) if was.get("type") == node.get("type") else None + if any(k != "label" and not (k == sql_key and self._same_graph_sql(_graph_sql(was), _graph_sql(node))) + for k in keys): changed[node_id] = "config" for node_id, node in before.items(): if node_id not in after: nodes.append(entry(node_id, node, "removed", [])) changed[node_id] = "removed" - same_edges = _edges(self.base.graph) == _edges(self.target.graph) - same_defaults = (self.base.graph.get("defaults") or {}) == (self.target.graph.get("defaults") or {}) + edges_before, edges_after = _edges(self.base.graph), _edges(self.target.graph) + for _, target_id, _ in edges_before ^ edges_after: + if target_id in before and target_id in after: + changed.setdefault(target_id, "edges") + defaults_before = self.base.graph.get("defaults") or {} + defaults_after = self.target.graph.get("defaults") or {} + differing = [k for k in dict.fromkeys([*defaults_before, *defaults_after]) + if defaults_before.get(k) != defaults_after.get(k) and k not in _QUIET_DEFAULTS] + for node_id, node in after.items(): + was = before.get(node_id) + if differing and was is not None and str(node.get("type")) in _MODEL_TYPES and any( + _unset(_config(node).get(k)) and _unset(_config(was).get(k)) for k in differing): + changed.setdefault(node_id, "defaults") + same_edges = edges_before == edges_after + same_defaults = defaults_before == defaults_after if not same_edges: self.notes.append(EDGES_CHANGED) if not same_defaults: @@ -828,33 +1108,46 @@ class _Compare: out.update(classification=cls, reasons=[f"{side}的口径卡「{self.title(card_id)}」里没有这个指标"]) return out assert bm is not None and tm is not None and b is not None and t is not None - caliber = _caliber_reasons(bm, tm, b.content or {}, t.content or {}) + defs = (_definition(self.base.graph, card_id, mid), _definition(self.target.graph, card_id, mid)) + caliber = _caliber_reasons(bm, tm, b.content or {}, t.content or {}, defs) + if shown := _display_change(bm, tm, defs): + self.notes.append(f"口径卡「{self.title(card_id)}」的指标「{out['name']}」{shown}，只影响显示，不算口径变化") self._delta(out, bm, tm) lineage = isinstance(bm.get("inputs"), list) and isinstance(tm.get("inputs"), list) + moved = not recompute._same(bm.get("value"), tm.get("value")) + changes: dict[str, _Change] = {} + if lineage and bp is not None and tp is not None: + changes = {p: self.provenance(p, bp.inputs.get(p), tp.inputs.get(p), moved=moved) + for p in dict.fromkeys([*tp.inputs, *bp.inputs])} if caliber: - out.update(classification=CALIBER_AND_DATA, reasons=caliber) - out["decomposition"], out["decomposition_note"] = self._decompose(bm, tm, b, t, bp, tp) + # 口径优先；喂它的取数也变了、说不清的，接在口径的说明后面照样写出来，不因为口径变了就不提。只在一边才有 + # 的输入是改表达式带来的，口径的说明已经说了 + both = set(bp.inputs) & set(tp.inputs) if bp is not None and tp is not None else set() + also = [c.reason for p, c in changes.items() if p in both and c.kind in ("sql", "template", "unknown") + and c.reason] + out.update(classification=CALIBER_AND_DATA, reasons=list(dict.fromkeys([*caliber, *also]))) + self._decompose(out, bm, tm, b, t, bp, tp) elif not lineage: quiet = self.template["same_graph"] and all(self.shape(x, y).structure == "same" for x, y in self.pairs) - same = recompute._same(bm.get("value"), tm.get("value")) - out.update(classification=UNCHANGED if quiet and same else INCOMPARABLE, - reasons=[] if quiet and same else [NO_LINEAGE]) + out.update(classification=UNCHANGED if quiet and not moved else INCOMPARABLE, + reasons=[] if quiet and not moved else [NO_LINEAGE]) else: - paths = list(dict.fromkeys([*tp.inputs, *bp.inputs])) - changes = [self.provenance(p, bp.inputs.get(p), tp.inputs.get(p)) for p in paths] - worst = _worst(changes) - reasons = list(dict.fromkeys(c.reason for c in changes if c.kind == worst.kind and c.reason)) + worst = _worst(changes.values()) + reasons = list(dict.fromkeys(c.reason for c in changes.values() if c.kind == worst.kind and c.reason)) if worst.kind in ("sql", "template"): out.update(classification=SQL_CHANGED, reasons=reasons, decomposition_note=SQL_NOT_SPLIT) elif worst.kind == "unknown": out.update(classification=INCOMPARABLE, reasons=reasons) elif worst.kind == "params": out.update(classification=PARAMS_ONLY, reasons=reasons) - elif recompute._same(bm.get("value"), tm.get("value")): + elif not moved: out.update(classification=UNCHANGED) + elif all(inp.snapshot for inp in tp.inputs.values()) and tp.inputs: + out.update(classification=DATA_ONLY, reasons=[DATA_FROM_QUERIES]) else: - linked = all(inp.snapshot for inp in tp.inputs.values()) and bool(tp.inputs) - out.update(classification=DATA_ONLY, reasons=[DATA_FROM_QUERIES if linked else DATA_FROM_UPSTREAM]) + # 「模板没有变化」只在整张图都没变时说；图有别的改动、都不在它的上游的，只说它的上游没变 + out.update(classification=DATA_ONLY, reasons=[ + DATA_FROM_UPSTREAM if self.template["same_graph"] else DATA_FROM_UPSTREAM_SCOPED]) for side, metric in (("基准运行", bm), ("这次运行", tm)): if metric.get("status") == "missing_input": out["reasons"].append(f"{side}这个指标缺少输入，没有值") @@ -872,11 +1165,21 @@ class _Compare: whole = all(isinstance(m.get("value"), int) for m in (bm, tm)) delta = _plain(t - b, whole=whole) meta = {**tm, "decimals": _places(bm, tm)} - out.update(delta=delta, delta_rendered=_signed(meta, delta) if delta != 0 else None, - delta_pct=_plain(round((t - b) / abs(b) * 100, 2), whole=False) if b != 0 else None) + out.update(delta=delta, delta_rendered=_signed(meta, delta) if delta != 0 else None, delta_pct=_percent(b, t)) - def _decompose(self, bm: dict[str, Any], tm: dict[str, Any], b: _Card, t: _Card, bp: Any, - tp: Any) -> tuple[dict[str, Any] | None, str | None]: + def _decompose(self, out: dict[str, Any], bm: dict[str, Any], tm: dict[str, Any], b: _Card, t: _Card, bp: Any, + tp: Any) -> None: + """严格拆分，写进 out 的 decomposition（拆不了的原因写进 decomposition_note）。为了两效应之和在 JSON 里也逐位 + 等于总变化，总变化可能按拆分的精度重写一遍（_split_exact）。""" + split, note = self._split(bm, tm, b, t, bp, tp) + out["decomposition"], out["decomposition_note"] = split, note + if split is not None and "delta" in split: + delta = split.pop("delta") + meta = {**tm, "decimals": _places(bm, tm)} + out.update(delta=delta, delta_rendered=_signed(meta, delta) if delta != 0 else None) + + def _split(self, bm: dict[str, Any], tm: dict[str, Any], b: _Card, t: _Card, bp: Any, + tp: Any) -> tuple[dict[str, Any] | None, str | None]: """严格拆分：(拆分, None) 或 (None, 为什么没法拆)。""" old, new = _exact(bm.get("value")), _exact(tm.get("value")) if old is None or new is None: @@ -899,8 +1202,10 @@ class _Compare: return None, SPLIT_MASKED for path in leaves: change = self.provenance(path, bp.inputs.get(path), t.inputs[path]) - if change.kind in ("sql", "template", "unknown"): + if change.kind in ("sql", "template"): return None, f"旧口径用到的输入取数也有变化，无法严格拆分：{change.reason}" + if change.kind == "unknown": + return None, f"无法确认旧口径用到的输入只是数据有变化，无法严格拆分：{change.reason}" # f_old(x_new)：旧表达式配这次的输入，按推演重算的规则还原、求值，取整按旧口径卡的小数位 synthetic = {"id": bm.get("id"), "expression": bm.get("expression"), "decimals": bm.get("decimals"), "inputs": [inp.item for inp in used], "substituted": None, "value": None} @@ -918,10 +1223,14 @@ class _Compare: if middle is None: return None, "按旧口径在这次的数据上算出的不是有限的数值，无法严格拆分" whole = all(isinstance(v, int) and not isinstance(v, bool) for v in (bm.get("value"), tm.get("value"), value)) - caliber, data = _plain(new - middle, whole=whole), _plain(middle - old, whole=whole) + total, caliber_d, data_d = _split_exact(new - old, new - middle) + caliber, data = _plain(caliber_d, whole=whole), _plain(data_d, whole=whole) meta = {**tm, "decimals": _places(bm, tm)} - return {"caliber_effect": caliber, "data_effect": data, "caliber_rendered": _signed(meta, caliber), - "data_rendered": _signed(meta, data)}, None + split = {"caliber_effect": caliber, "data_effect": data, "caliber_rendered": _signed(meta, caliber), + "data_rendered": _signed(meta, data)} + if total != new - old: + split["delta"] = _plain(total, whole=whole) + return split, None def _inputs(self, bp: Any, tp: Any) -> list[dict[str, Any]]: """两边的输入按路径对齐：值（遮罩列、核对不了遮没遮的不给），各自落在的查询快照。""" @@ -961,13 +1270,15 @@ class _Compare: return next((r for r in found if r["rows"]), found[0] if found else None) def _attribute(self, b: Any, t: Any) -> dict[str, Any] | None: - """一个输入的维度归因：{columns, measure, rows, total_delta, note}；归不了因的照实写在 note 里（rows 为空）。 - 快照里没有可以当维度的列（只有一行汇总这种）返回 None：没什么可归的。""" + """一个输入的维度归因：{path, base_artifact, target_artifact, columns, measure, rows, total_delta, note}； + 归不了因的照实写在 note 里（rows 为空）。快照里没有可以当维度的列（只有一行汇总这种）返回 None：没什么可归的。""" measure = _locator(b).get("column") if not isinstance(measure, str) or _locator(t).get("column") != measure: return None sb, st = self._snap(self.base_queries, b), self._snap(self.target_queries, t) - empty = {"columns": [], "measure": measure, "rows": [], "total_delta": None} + # 归因用的是哪个输入、两边各自哪份快照：一个指标有几个落在快照上的输入时分得清 + trace = {"path": t.path, "base_artifact": b.snapshot, "target_artifact": t.snapshot} + empty = {**trace, "columns": [], "measure": measure, "rows": [], "total_delta": None} if sb is None or st is None: return {**empty, "note": UNREADABLE_SNAPSHOT} for side, snap in (("基准运行", sb), ("这次运行", st)): @@ -1016,7 +1327,7 @@ class _Compare: rows.sort(key=lambda r: -r[0]) # 稳定排序：变化一样大的按先出现的 if len(rows) > TOP_KEYS: notes.append(MORE_KEYS.format(count=len(rows), top=TOP_KEYS)) - return {"columns": dims, "measure": measure, "rows": [r for _, r in rows[:TOP_KEYS]], + return {**trace, "columns": dims, "measure": measure, "rows": [r for _, r in rows[:TOP_KEYS]], "total_delta": _plain(total, whole=False), "note": "；".join(notes) or None} def _snap(self, queries: list[_Query], inp: Any) -> dict[str, Any] | None: @@ -1043,26 +1354,50 @@ def _rendered(metric: dict[str, Any] | None) -> str | None: return rendered if isinstance(rendered, str) else recompute._render(metric, metric.get("value")) -def _caliber_reasons(bm: dict[str, Any], tm: dict[str, Any], bc: dict[str, Any], tc: dict[str, Any]) -> list[str]: - """口径变了的地方，一处一句；没变返回空列表。""" +def _setting(metric: dict[str, Any], definition: dict[str, Any] | None, key: str) -> tuple[bool, Any]: + """指标的小数位（decimals）或显示格式（format）：(知不知道, 值)。卡上记了的按卡上的；升级前的卡没记这两项（当时 + 按定义里的小数位取过整），按那次运行图里的指标定义（和当时一样规整）；定义也找不到的不知道，不拿来比。""" + if key in metric: + value = metric[key] + return True, (value or DEFAULT_FORMAT) if key == "format" else value + if definition is None: + return False, None + display, problem = _display_of(definition) + return (False, None) if problem else (True, display[key]) + + +def _caliber_reasons(bm: dict[str, Any], tm: dict[str, Any], bc: dict[str, Any], tc: dict[str, Any], + defs: tuple[dict[str, Any] | None, dict[str, Any] | None] = (None, None)) -> list[str]: + """口径变了的地方，一处一句；没变返回空列表。defs 是两次运行图里这个指标的定义（老卡没记小数位、格式时用）。""" out = [] if _expression(bm) != _expression(tm): out.append(f"表达式由「{_shown(bm.get('expression'))}」改为「{_shown(tm.get('expression'))}」") before, after = str(bc.get("caliber_version") or ""), str(tc.get("caliber_version") or "") if before != after: out.append(f"口径版本由 {before or '未设置'} 改为 {after or '未设置'}") - if bm.get("decimals") != tm.get("decimals"): + (kb, db), (kt, dt) = _setting(bm, defs[0], "decimals"), _setting(tm, defs[1], "decimals") + if kb and kt and db != dt: def places(d: Any) -> str: return "未设置" if d is None else str(d) - out.append(f"小数位由 {places(bm.get('decimals'))} 改为 {places(tm.get('decimals'))}") + out.append(f"小数位由 {places(db)} 改为 {places(dt)}") if (bm.get("unit") or "") != (tm.get("unit") or ""): out.append(f"单位由「{bm.get('unit') or '无'}」改为「{tm.get('unit') or '无'}」") - fb, ft = bm.get("format") or DEFAULT_FORMAT, tm.get("format") or DEFAULT_FORMAT - if fb != ft: + (kb, fb), (kt, ft) = _setting(bm, defs[0], "format"), _setting(tm, defs[1], "format") + if kb and kt and (fb == _RATIO) != (ft == _RATIO): + # 比率（0.087 显示成 8.7%）和普通数值的含义不同；千分位、不分组只是写法，见 _display_change out.append(f"显示格式由「{_FORMAT_LABEL.get(fb, fb)}」改为「{_FORMAT_LABEL.get(ft, ft)}」") return out +def _display_change(bm: dict[str, Any], tm: dict[str, Any], + defs: tuple[dict[str, Any] | None, dict[str, Any] | None]) -> str | None: + """只影响显示的格式变化（千分位和不分组互换）：一句说明；没有返回 None。""" + (kb, fb), (kt, ft) = _setting(bm, defs[0], "format"), _setting(tm, defs[1], "format") + if not (kb and kt) or fb == ft or _RATIO in (fb, ft): + return None + return f"显示格式由「{_FORMAT_LABEL.get(fb, fb)}」改为「{_FORMAT_LABEL.get(ft, ft)}」" + + def _input_value(inp: Any) -> Any: """输入在对比里给人看的值：遮罩列、落在读不出来的快照上的不给；容器写卡上记的大概。""" if inp is None or inp.masked or (inp.unreadable and inp.shape == "scalar"): @@ -1077,6 +1412,45 @@ def _locator(inp: Any) -> dict[str, Any]: return locator if isinstance(locator, dict) else {} +def _same_input(b: Any, t: Any) -> bool | None: + """一个输入两边的值相不相同。容器按还原出来的完整值比（卡上只记了「[2 项]」这样的大概，长度相同、内容不同的 + 不能当一样）；完整值还原不出来时，大概不同的就是不同，大概相同的说不清，返回 None。""" + if b.shape == "container" or t.shape == "container": + if b.shape == t.shape == "container" and b.problem is None and t.problem is None: + return recompute._same(b.value, t.value) + return None if recompute._same(b.item.get("value"), t.item.get("value")) else False + return recompute._same(b.item.get("value"), t.item.get("value")) + + +def _whole(inp: Any, q: _Query) -> bool: + """这个输入引用的是整份查询结果：数组字段的行段从第一行到最后一行，或者容器本身就是整份结果。""" + if inp.table: + return True + return inp.rows is not None and q.snap is not None and inp.rows == (0, len(q.snap["rows"]) - 1) + + +def _rows_text(inp: Any) -> str: + if inp.row is not None: + return f"第 {inp.row + 1} 行" + if inp.rows is not None: + return f"第 {inp.rows[0] + 1} 至 {inp.rows[1] + 1} 行" + return "整份结果" + + +def _columns_text(columns: Any) -> str: + if isinstance(columns, dict): + return "「" + "、".join(str(c) for c in columns.values()) + "」" + return f"「{columns}」" if columns else "（未记录）" + + +def _merge_inputs(q: _Query) -> dict[str, str] | None: + """合并查询的输入：别名 → 上游快照。没记录的返回 None。""" + if q.inputs is None: + return None + return {str(i["alias"]): str(i["artifact"]) for i in q.inputs + if isinstance(i, dict) and i.get("alias") and i.get("artifact")} + + def _records(snap: dict[str, Any], inp: Any) -> list[tuple[int, dict[str, Any]]] | None: """归因用到的行：(行号, {列: 值})。数组字段只用它那一段行，单格用整份结果。""" columns = [str(c) for c in snap.get("columns") or []] diff --git a/backend/tests/fixtures/compare_runs.py b/backend/tests/fixtures/compare_runs.py index 1d8bb6e2..a961018e 100644 --- a/backend/tests/fixtures/compare_runs.py +++ b/backend/tests/fixtures/compare_runs.py @@ -12,6 +12,7 @@ from app.core.artifact_store import canonical_json, content_hash from app.engine import compare, recompute from app.engine.evidence import render_number from app.engine.expressions import parse_expression, substitute +from app.engine.merge_query import MERGE_SOURCE TOOL = "db_query__shop" WEEK36 = ("2026-08-31", "2026-09-06") @@ -110,15 +111,15 @@ class Fake: return artifact def agent(self, node_id, sql, columns=COLUMNS, rows=BASE_ROWS, *, types=TYPES, truncated=False, masks=None, - verify=()): - """Agent 节点自己查一次库，按出处交回数组字段 amounts（快照 gmv 列的全部行）。verify 是同一节点里核查类的 - 查询（role: verify，不进台账）。""" + verify=(), column="gmv", span=None): + """Agent 节点自己查一次库，按出处交回数组字段 amounts（默认是快照 gmv 列的全部行；column、span 改引用的列和 + 行段）。verify 是同一节点里核查类的查询（role: verify，不进台账）。""" artifact = self.snapshot(sql, columns, rows, truncated=truncated, types=types, masks=masks) - last = len(rows) - 1 - gmv = columns.index("gmv") - data = {"amounts": [r[gmv] for r in rows]} - evidence = {"amounts": {"ref": f"Q1.r0-{last}.gmv", "call": "Q1", "artifact": artifact, - "locator": {"rows": [0, last], "column": "gmv"}, "status": "verified"}} + first, last = span or (0, len(rows) - 1) + at = columns.index(column) + data = {"amounts": [r[at] for r in rows[first:last + 1]]} + evidence = {"amounts": {"ref": f"Q1.r{first}-{last}.{column}", "call": "Q1", "artifact": artifact, + "locator": {"rows": [first, last], "column": column}, "status": "verified"}} output = self.put({"text": "按区域列出了金额", "data": data, "data_evidence": evidence}) self.add("node.started", node_id, {"node_type": "agent"}) self.add("tool.start", node_id, {"tool": TOOL, "args": {"sql": sql}, "call_id": "c1"}) @@ -130,6 +131,40 @@ class Fake: self.entry(node_id, artifact, columns, rows, call_id="c1", truncated=truncated)]}) return artifact + def plain_agent(self, node_id, sqls, data, columns=COLUMNS, rows=BASE_ROWS, *, types=TYPES): + """没开按出处核对的 Agent：查几次库，交回的 data 没有出处（data_evidence），口径卡只知道是它写的。""" + self.add("node.started", node_id, {"node_type": "agent"}) + entries = [] + for i, sql in enumerate(sqls): + artifact = self.snapshot(sql, columns, rows, types=types) + self.add("tool.start", node_id, {"tool": TOOL, "args": {"sql": sql}, "call_id": f"c{i}"}) + self.add("tool.end", node_id, {"tool": TOOL, "call_id": f"c{i}", "duration_ms": 5, + "query_artifact": artifact}) + entries.append(self.entry(node_id, artifact, columns, rows, call_id=f"c{i}")) + output = self.put({"text": "按区域分析了金额", "data": data}) + self.add("node.finished", node_id, {"duration_ms": 9, "attempt": 1, "artifact": output, "evidence": entries}) + + def merge(self, node_id, sql, inputs, columns, rows, *, types=None): + """合并查询：几份上游快照按别名落进内存库，再执行一条合并 SQL。快照里记着各个输入（别名、节点、快照）， + 事件照合并节点的形状写（merge.end 加 node.finished 台账里的 query 条目，没有 tool.end）。 + inputs 是 [(别名, 上游节点, 上游快照)]。""" + snap = {"columns": columns, "rows": rows, "row_count": len(rows), "truncated": False, "sql": sql, + "source": MERGE_SOURCE, "sources": [self.source], + "inputs": [{"alias": alias, "node_id": upstream, "artifact": artifact, + "rows": len(self.store[artifact]["rows"]), "columns": self.store[artifact]["columns"], + "source": self.store[artifact].get("source")} for alias, upstream, artifact in inputs], + "warnings": [], **({"column_types": types} if types else {})} + artifact = self.put(snap) + self.queries.append(artifact) + output = self.put({**snap, "artifact": artifact}) + view = [{k: v for k, v in item.items() if k != "columns"} for item in snap["inputs"]] + self.add("node.started", node_id, {"node_type": "merge"}) + self.add("merge.end", node_id, {"inputs": view, "sql": sql, "rows": len(rows), "query_artifact": artifact}) + self.add("node.finished", node_id, {"duration_ms": 2, "attempt": 1, "artifact": output, "evidence": [ + {"kind": "query", "node_id": node_id, "exec": 1, "artifact": artifact, "source": MERGE_SOURCE, + "columns": columns, "rows": len(rows), "truncated": False}]}) + return artifact + def code(self, node_id, value, var): """沙箱代码节点算出一个数写进变量：它不查库，对不上具体的查询。""" output = self.put({var: value}) @@ -169,11 +204,12 @@ def cell_in(row, column, value, artifact, node_id="fetch"): "eid": f"ev:cell:{artifact[:8]}:{row}:{column}", "status": "verified"} -def amounts_in(artifact, rows): - last = len(rows) - 1 - return {"path": "vars.kpi.amounts", "value": f"[{len(rows)} 项]", "node_id": "bot", "via": "agent_field", - "field": "amounts", "ref": f"Q1.r0-{last}.gmv", "artifact": artifact, - "locator": {"rows": [0, last], "column": "gmv"}, "eid": f"ev:rows:{artifact[:8]}", "status": "verified"} +def amounts_in(artifact, rows, *, column="gmv", span=None): + first, last = span or (0, len(rows) - 1) + return {"path": "vars.kpi.amounts", "value": f"[{last - first + 1} 项]", "node_id": "bot", "via": "agent_field", + "field": "amounts", "ref": f"Q1.r{first}-{last}.{column}", "artifact": artifact, + "locator": {"rows": [first, last], "column": column}, "eid": f"ev:rows:{artifact[:8]}:{first}:{column}", + "status": "verified"} def scalar_in(path, value, node_id, via): diff --git a/backend/tests/test_compare_api.py b/backend/tests/test_compare_api.py index cd2cf0d9..eda3efdf 100644 --- a/backend/tests/test_compare_api.py +++ b/backend/tests/test_compare_api.py @@ -149,12 +149,14 @@ async def test_unsealed_runs_are_refused(client): seal=False) body = await get(client, f"/api/runs/{target_id}/compare?base={base_id}", 409) assert body["code"] == "compare_unsealed" and body["detail"].startswith("这次运行") + # 已经结束、只是没有封存清单的：说没有封存清单，不说「运行结束后才能对比」 + assert "没有封存清单" in body["detail"] and "尚未结束" not in body["detail"] body = await get(client, f"/api/runs/{base_id}/compare?base={target_id}", 409) assert body["code"] == "compare_unsealed" and body["detail"].startswith("作为基准的运行") # 封存过、又被接着运行（还在运行）的：封存范围之后还会变，同样不比 running = await persist(week("r3", NEXT_ROWS, source=SOURCE), workflow, status="running") body = await get(client, f"/api/runs/{running}/compare?base={base_id}", 409) - assert body["code"] == "compare_unsealed" + assert body["code"] == "compare_unsealed" and "尚未结束" in body["detail"] async def test_a_broken_seal_is_refused(client): @@ -233,12 +235,15 @@ async def test_masked_columns_never_leave_the_server(client): # -------------------------------------------------------------------------- -async def bare(workflow_id, finished, *, status="succeeded", run_class="formal", schedule_id=None, sealed=True) -> str: - """只为挑基准用的运行记录：挑基准只看运行记录上的几列。""" +async def bare(workflow_id, finished, *, status="succeeded", run_class="formal", schedule_id=None, sealed=True, + resumed=False) -> str: + """只为挑基准用的运行记录：挑基准只看运行记录上的几列。resumed：结束、封存过又被接着运行（状态已回到运行中， + 结束时间和封存还没清掉），只能靠状态把它挡在外面。""" + done = status in ("succeeded", "failed", "cancelled") or resumed async with SessionLocal() as session: run = Run(workflow_id=workflow_id, workflow_name="区域周报", status=status, run_class=run_class, schedule_id=schedule_id, started_at=finished - timedelta(minutes=2), - finished_at=finished if status in ("succeeded", "failed", "cancelled") else None, + finished_at=finished if done else None, manifest_hash="ab" * 32 if sealed else None, manifest_seq=1 if sealed else None) session.add(run) await session.commit() @@ -271,6 +276,7 @@ async def test_the_default_base_skips_what_cannot_be_compared(client): good = await bare(workflow, T0) await bare(workflow, T0 + day, sealed=False) # 没封存 await bare(workflow, T0 + 2 * day, status="running") # 还在运行 + await bare(workflow, T0 + 2 * day, status="running", resumed=True) # 结束过、又被接着运行 await bare(workflow, T0 + 3 * day, run_class="exploratory") # 别的运行类别 await bare(await new_workflow("另一个周报"), T0 + 4 * day) # 别的工作流 current = await bare(workflow, T0 + 5 * day) diff --git a/backend/tests/test_compare_engine.py b/backend/tests/test_compare_engine.py index 1d2d9f3f..328b0044 100644 --- a/backend/tests/test_compare_engine.py +++ b/backend/tests/test_compare_engine.py @@ -9,9 +9,12 @@ SQL，Agent 节点 bot 自己再查一次、把各区域的金额作为数组字 为 1；对不上时占比为空并说明；截断的快照不归因，照实写 - 遮罩列的值绝不出现在结果里；用到遮罩列的指标不拆分 - 模板对比：新增、删除的节点，配置有变化的节点和配置项名 +- 输入的出处：合并查询追到它的上游查询；Agent 按出处交的字段还要引用同一列、同一段行；没按出处交的数、上游数据 + 没变数却变了的，说不清变化从哪来（incomparable）；图里写死的字面量改了、默认设置和连线变了都算模板变化 """ from __future__ import annotations +import copy import json from decimal import Decimal @@ -19,13 +22,16 @@ from app.engine import compare from app.engine.merge_query import MERGE_SOURCE from tests.fixtures.compare_runs import ( BASE_ROWS, + COLUMNS, EAST, NEXT_ROWS, REFUND, + TOOL, WEEK37, Fake, amounts_in, cell_in, + chain, default_metrics, metric, node, @@ -167,14 +173,14 @@ def test_the_split_adds_up_exactly_with_awkward_decimals(): return make base_rows = [["东区", 1000.1, 0.3], *BASE_ROWS[1:]] - next_rows = [["东区", 1000.7, 0.2], *NEXT_ROWS[1:]] + next_rows = [["东区", 1000.75, 0.2], *NEXT_ROWS[1:]] base = week("r1", base_rows, metrics=build(f"{EAST} * 0.1 - {REFUND}", 2)) target = week("r2", next_rows, metrics=build(f"{EAST} * 0.3 + {REFUND}", 3), version="v2") ratio = by_id(run_compare(base, target))["ratio"] split = ratio["decomposition"] assert split is not None, ratio["decomposition_note"] - # f_old(x_new) 按旧口径的小数位取整：round(1000.7 * 0.1 - 0.2, 2) = 99.87 - assert exact(split["data_effect"]) == exact(99.87) - exact(ratio["base"]) + # f_old(x_new) 按旧口径的小数位取整：1000.75 * 0.1 - 0.2 = 99.875，取 2 位是 99.88（按新口径的 3 位是 99.875） + assert exact(split["data_effect"]) == exact(99.88) - exact(ratio["base"]) assert exact(split["caliber_effect"]) + exact(split["data_effect"]) == exact(ratio["delta"]) assert exact(ratio["delta"]) == exact(ratio["target"]) - exact(ratio["base"]) @@ -204,6 +210,20 @@ def test_no_split_when_the_sql_changed_too(): net = by_id(run_compare(base, target))["net"] assert net["classification"] == "caliber_and_data" and net["decomposition"] is None assert "SQL" in net["decomposition_note"] + # 两条不同的查询的行不拿来按维度配对 + assert net["attribution"] is None + + +def test_a_caliber_change_still_names_the_sql_change(): + """口径变了、SQL 结构也变了、旧口径的叶子这次又没记录：分类按口径，SQL 的变化照样写进说明。""" + base = week("r1", metrics=net_metrics(f"{EAST} - {REFUND}"), version="v1") + paid = region_sql(extra=" AND status = 'paid'") + target = week("r2", NEXT_ROWS, sql=paid, metrics=net_metrics(EAST), version="v2") + net = by_id(run_compare(base, target))["net"] + assert net["classification"] == "caliber_and_data" and net["decomposition"] is None + assert any("区域取数" in r and "SQL" in r for r in net["reasons"]), net["reasons"] + # 退款那一格只在旧口径里用到：改表达式带来的，不再另说一遍「只在一边有记录」 + assert not any("只在一边" in r for r in net["reasons"]), net["reasons"] def test_a_unit_change_is_not_subtracted(): @@ -217,15 +237,102 @@ def test_a_unit_change_is_not_subtracted(): assert "单位" in east["decomposition_note"] +def test_display_settings_that_change_the_value_are_caliber_changes(): + def build(**kw): + def make(fetch, bot, rows, bot_rows): + return [metric("east", EAST, rows[0][1], [cell_in(0, "gmv", rows[0][1], fetch)], **kw)] + return make + + def east(before, after): + return by_id(run_compare(week("r1", metrics=build(**before)), week("r2", NEXT_ROWS, metrics=build(**after)))) + + # 小数位改变取整 + found = east({"decimals": 1}, {"decimals": 0}) + assert found["east"]["classification"] == "caliber_and_data" and any("小数位" in r for r in found["east"]["reasons"]) + # 比率与否改变数值的含义 + found = east({"unit": "", "fmt": "plain"}, {"unit": "", "fmt": "percent_of_ratio"}) + assert found["east"]["classification"] == "caliber_and_data" and found["east"]["delta"] is None + # 千分位和不分组只影响显示：分类照数据算，变化写进说明 + result = run_compare(week("r1", metrics=build(fmt="thousands")), week("r2", NEXT_ROWS, metrics=build(fmt="plain"))) + assert by_id(result)["east"]["classification"] == "data_only" + assert any("显示格式" in n for n in result["notes"]) + + +def test_an_old_card_without_display_settings_is_not_a_caliber_change(): + """升级前封存的运行：卡上的指标没记小数位、显示格式和输入（当时按定义里的小数位取过整）。""" + def old(run_id, value, graph=None): + run = Fake(run_id, graph=graph) + run.tool("fetch", region_sql()) + run.agent("bot", region_sql()) + run.card([{"id": "east", "name": "东区金额", "unit": "元", "value": value, "expression": EAST}]) + return run + + def new(run_id, value): + run = Fake(run_id) + fetch = run.tool("fetch", region_sql()) + run.agent("bot", region_sql()) + run.card([metric("east", EAST, value, [cell_in(0, "gmv", value, fetch)], name="东区金额")]) + return run + + east = by_id(run_compare(old("r1", 1200.5), new("r2", 1200.5)))["east"] + assert (east["classification"], east["reasons"]) == ("unchanged", []) + east = by_id(run_compare(old("r1", 1200.5), new("r2", 1500.5)))["east"] + assert east["classification"] == "incomparable" and east["reasons"] == [compare.NO_LINEAGE] + # 那次运行的图里写着定义：按定义里的小数位核对 + graph = weekly_graph() + graph["nodes"][3]["data"]["config"]["metrics"] = [{"id": "east", "expression": EAST, "decimals": 2}] + east = by_id(run_compare(old("r1", 1200.5, graph), new("r2", 1200.5)))["east"] + assert east["classification"] == "caliber_and_data" and any("小数位由 2 改为 1" in r for r in east["reasons"]) + + +def test_the_split_adds_up_in_json_without_fixed_decimals(): + """没设小数位的指标：三个数各自转成浮点也要逐位相加（接口按 JSON 交回的就是浮点）。""" + def build(expression): + def make(fetch, bot, rows, bot_rows): + gmv, refund = rows[0][1], rows[0][2] + inputs = [cell_in(0, "gmv", gmv, fetch), cell_in(0, "refund", refund, fetch)] + value = eval(expression.replace(EAST, repr(gmv)).replace(REFUND, repr(refund))) + return [metric("m", expression, value, inputs, decimals=None, unit="")] + return make + + base_rows = [["东区", 0.1, 0.2], *BASE_ROWS[1:]] + next_rows = [["东区", 0.7, 0.1], *NEXT_ROWS[1:]] + base = week("r1", base_rows, metrics=build(f"{EAST} + {REFUND}")) # 0.30000000000000004 + target = week("r2", next_rows, metrics=build(f"{EAST} * 3 + {REFUND}"), version="v2") # 2.1999999999999997 + m = by_id(run_compare(base, target))["m"] + split = m["decomposition"] + assert split is not None, m["decomposition_note"] + assert exact(split["caliber_effect"]) + exact(split["data_effect"]) == exact(m["delta"]) + assert abs(m["delta"] - (m["target"] - m["base"])) < 1e-12 + + +def test_a_tiny_base_does_not_break_the_percentage(): + """基准是浮点误差那么小的数、这次很大：百分比写得出来（或者为空），对比不能整个失败。""" + def build(value): + def make(fetch, bot, rows, bot_rows): + m = metric("diff", EAST, 1.0, [cell_in(0, "gmv", rows[0][1], fetch)], decimals=None) + m["value"], m["rendered"] = value, "—" + return [m] + return make + diff = by_id(run_compare(week("r1", metrics=build(0.30000000000000004 - 0.3)), + week("r2", NEXT_ROWS, metrics=build(1e8))))["diff"] + assert diff["delta"] == 1e8 - (0.30000000000000004 - 0.3) + assert diff["delta_pct"] is not None and diff["delta_pct"] > 1e25 + + # -------------------------------------------------------------------------- # 维度归因 # -------------------------------------------------------------------------- def test_attribution_pairs_keys_and_shares_add_up_to_one(): - total = by_id(run_compare(week("r1"), week("r2", NEXT_ROWS)))["total"] + base, target = week("r1"), week("r2", NEXT_ROWS) + total = by_id(run_compare(base, target))["total"] attribution = total["attribution"] assert attribution["columns"] == ["region"] and attribution["measure"] == "gmv" + # 归因用的是哪个输入、两边各自哪份快照 + assert (attribution["path"], attribution["base_artifact"], attribution["target_artifact"]) == ( + "vars.kpi.amounts", base.queries[1], target.queries[1]) assert attribution["total_delta"] == 200.0 rows = attribution["rows"] assert [(r["key"], r["base"], r["target"], r["delta"]) for r in rows] == [ @@ -265,6 +372,49 @@ def test_no_share_when_the_measure_does_not_add_up_to_the_input(): assert all(r["share"] is None for r in attribution["rows"]) and attribution["note"] +def test_no_share_when_only_one_side_adds_up(): + """基准运行里东区就是全部（各区域之和等于它），这次南区也有了：只有一边对得上，不给占比。""" + base_rows = [["东区", 100.0, 0.0], ["南区", 0.0, 0.0], ["西区", 0.0, 0.0]] + next_rows = [["东区", 150.0, 0.0], ["南区", 20.0, 0.0], ["西区", 0.0, 0.0]] + east = by_id(run_compare(week("r1", base_rows), week("r2", next_rows)))["east"] + attribution = east["attribution"] + assert east["classification"] == "data_only" and len(attribution["rows"]) == 3 + assert all(r["share"] is None for r in attribution["rows"]) and compare.NO_SHARE in attribution["note"] + + +def test_a_total_row_summed_in_floating_point_still_adds_up(): + """合计行是库里按浮点加出来的（0.1 + 0.2 + 0.3 = 0.6000000000000001）：差在最后几位也算相等，照给占比。""" + base_rows = [["东区", 0.1, 0.0], ["南区", 0.2, 0.0], ["西区", 0.3, 0.0], ["合计", 0.1 + 0.2 + 0.3, 0.0]] + next_rows = [["东区", 0.4, 0.0], ["南区", 0.2, 0.0], ["西区", 0.3, 0.0], ["合计", 0.4 + 0.2 + 0.3, 0.0]] + + def build(fetch, bot, rows, bot_rows): + return [metric("all", "cell(nodes.fetch, 3, 'gmv')", rows[3][1], [cell_in(3, "gmv", rows[3][1], fetch)], + decimals=None)] + found = by_id(run_compare(week("r1", base_rows, metrics=build), week("r2", next_rows, metrics=build)))["all"] + rows = found["attribution"]["rows"] + assert [r["key"] for r in rows] == [["东区"], ["南区"], ["西区"]] + assert abs(sum(r["share"] for r in rows) - 1) < 1e-9 + + +def test_date_typed_and_constant_columns_are_not_dimensions(): + """日期类型的列（两边有相同的日子也不算）和两边都只有一个值的列（币种）不当维度。""" + columns = ["day", "region", "currency", "gmv"] + types = {"day": "date", "region": "text", "currency": "text", "gmv": "number"} + + def make(run_id, rows): + run = Fake(run_id) + sql = "SELECT day, region, currency, SUM(amount) AS gmv FROM orders GROUP BY day, region, currency" + bot = run.agent("bot", sql, columns, rows, types=types) + run.card([metric("total", "sum(vars.kpi.amounts)", sum(r[3] for r in rows), [amounts_in(bot, rows)])]) + return run + + base_rows = [["2026-09-01", "东区", "CNY", 10.0], ["2026-09-02", "南区", "CNY", 20.0]] + next_rows = [["2026-09-02", "东区", "CNY", 15.0], ["2026-09-03", "南区", "CNY", 18.0]] + attribution = by_id(run_compare(make("r1", base_rows), make("r2", next_rows)))["total"]["attribution"] + assert attribution["columns"] == ["region"] + assert [(r["key"], r["delta"]) for r in attribution["rows"]] == [(["东区"], 5.0), (["南区"], -2.0)] + + def test_a_total_row_is_left_out_of_its_own_breakdown(): """合计行和各区域在同一份结果里：指标取合计那一格，按其余各行归因，占比之和为 1。""" base_rows = [*BASE_ROWS, ["合计", 2600.5, 150.0]] @@ -290,6 +440,16 @@ def test_a_truncated_snapshot_is_not_attributed(): assert result_queries(run_compare(base, target), "bot")["target_truncated"] is True +def test_a_truncated_base_snapshot_is_not_attributed_either(): + base = Fake("r1") + fetch = base.tool("fetch", region_sql()) + bot = base.agent("bot", region_sql(), truncated=True) + base.card(default_metrics(fetch, bot, BASE_ROWS, BASE_ROWS)) + total = by_id(run_compare(base, week("r2", NEXT_ROWS)))["total"] + assert total["classification"] == "data_only" + assert total["attribution"]["rows"] == [] and "基准运行" in total["attribution"]["note"] + + def result_queries(result, node_id): return next(q for q in result["queries"] if q["node_id"] == node_id) @@ -412,10 +572,49 @@ def test_an_input_from_a_code_node_follows_its_upstream(): # 上游什么都没变：变化来自数据 calc = by_id(run_compare(make("r1", BASE_ROWS, 10), make("r2", NEXT_ROWS, 12)))["calc"] assert calc["classification"] == "data_only" and calc["delta"] == 4 + assert calc["reasons"] == [compare.DATA_FROM_UPSTREAM] # 上游的 SQL 结构变了：说不清它变没变，不能报成数据变化 calc = by_id(run_compare(make("r1", BASE_ROWS, 10), make("r2", NEXT_ROWS, 12, sql=region_sql(extra=" AND status = 'paid'"))))["calc"] assert calc["classification"] == "incomparable" and any("调整" in r for r in calc["reasons"]) + # 上游查到的数据两次一模一样，它交的数却变了：变化不来自数据 + calc = by_id(run_compare(make("r1", BASE_ROWS, 10), make("r2", BASE_ROWS, 12)))["calc"] + assert calc["classification"] == "incomparable" and any("调整" in r for r in calc["reasons"]) + assert compare.DATA_FROM_UPSTREAM not in calc["reasons"] + + +def test_a_value_from_a_querying_node_that_moved_without_its_data_is_incomparable(): + """调用工具节点整份交出的结果（不是 cell() 取的格）：它查到的行两次一样，交出的值（这里是查询耗时）却变了， + 变化不来自数据；查到的行也变了的，照常是数据变化。""" + def make(run_id, elapsed, rows=BASE_ROWS): + run = Fake(run_id) + run.tool("fetch", region_sql(), rows=rows) + run.agent("bot", region_sql(), rows=rows) + run.card([metric("cost", "vars.res.elapsed_ms", elapsed, + [scalar_in("vars.res.elapsed_ms", elapsed, "fetch", "tool")], unit="毫秒", decimals=None)]) + return run + cost = by_id(run_compare(make("r1", 3), make("r2", 5)))["cost"] + assert cost["classification"] == "incomparable" and any("区域取数" in r for r in cost["reasons"]) + cost = by_id(run_compare(make("r1", 3), make("r2", 5, NEXT_ROWS)))["cost"] + assert cost["classification"] == "data_only" + + +def test_data_from_upstream_is_only_claimed_when_the_graph_is_the_same(): + """改了它下游的节点（成果字段）：它的上游没变，变化照样来自上游数据，但不能说「模板没有变化」。""" + def make(run_id, rows, value, graph): + run = Fake(run_id, graph=graph) + run.tool("fetch", region_sql(), rows=rows) + run.agent("bot", region_sql(), rows=rows) + run.code("calc", value, "adj") + run.card(code_metrics(value)(None, None, rows, rows)) + return run + + base_graph = weekly_graph(between=(node("calc", "code", "调整", code="print(1)", assign_to="adj"),)) + target_graph = copy.deepcopy(base_graph) + target_graph["nodes"][-1]["data"]["config"] = {"fields": [{"name": "周报", "value": "{{ nodes.write.text }}"}]} + calc = by_id(run_compare(make("r1", BASE_ROWS, 10, base_graph), make("r2", NEXT_ROWS, 12, target_graph)))["calc"] + assert calc["classification"] == "data_only" + assert calc["reasons"] == [compare.DATA_FROM_UPSTREAM_SCOPED] def test_the_code_node_itself_changed_is_a_template_change(): @@ -430,6 +629,133 @@ def test_the_code_node_itself_changed_is_a_template_change(): assert calc["classification"] == "sql_changed" and any("调整" in r for r in calc["reasons"]) +def test_rewired_edges_are_a_template_change(): + """调整节点改由取数节点接进来（各节点的配置都没动）：它交的数变了算模板变化，不能说变化来自上游数据。""" + calc_node = node("calc", "code", "调整", code="result = 1", assign_to="adj") + base_graph = weekly_graph(between=(calc_node,)) + target_graph = copy.deepcopy(base_graph) + target_graph["edges"] = [e for e in target_graph["edges"] if (e["source"], e["target"]) != ("bot", "calc")] + target_graph["edges"].append({"source": "fetch", "target": "calc"}) + + def make(run_id, graph, value): + run = Fake(run_id, graph=graph) + run.tool("fetch", region_sql()) + run.agent("bot", region_sql()) + run.code("calc", value, "adj") + run.card(code_metrics(value)(None, None, BASE_ROWS, BASE_ROWS)) + return run + result = run_compare(make("r1", base_graph, 10), make("r2", target_graph, 12)) + calc = by_id(result)["calc"] + assert calc["classification"] == "sql_changed" and any("连线" in r for r in calc["reasons"]), calc["reasons"] + assert compare.EDGES_CHANGED in result["notes"] + + +def uncited(run_id, share, *, graph=None, sqls=None): + """Agent 没开按出处核对，口径卡用了它交的占比 vars.kpi.share；东区金额照旧取取数节点的那一格。""" + run = Fake(run_id, graph=graph) + fetch = run.tool("fetch", region_sql()) + run.plain_agent("bot", sqls or [region_sql()], {"kpi": {"share": share}}) + run.card([metric("east", EAST, BASE_ROWS[0][1], [cell_in(0, "gmv", BASE_ROWS[0][1], fetch)], name="东区金额"), + metric("share", "vars.kpi.share * 100", share * 100, [scalar_in("vars.kpi.share", share, "bot", "agent")], + unit="%", fmt="plain")]) + return run + + +def test_a_number_the_agent_gave_without_citing_is_incomparable(): + """Agent 没按出处交的数：两次查到的数据一样，数却变了（模型本身的差异），不能说是数据变化。""" + result = run_compare(uncited("r1", 0.46), uncited("r2", 0.52)) + share = by_id(result)["share"] + assert share["classification"] == "incomparable" and share["attribution"] is None + assert any("区域分析" in r and "查询快照" in r for r in share["reasons"]), share["reasons"] + assert by_id(result)["east"]["classification"] == "unchanged" + # 交的数一样：照常没变 + assert by_id(run_compare(uncited("r1", 0.46), uncited("r2", 0.46)))["share"]["classification"] == "unchanged" + + +def test_an_extra_query_by_the_writer_is_a_sql_change(): + extra = "SELECT COUNT(*) AS n FROM refunds" + share = by_id(run_compare(uncited("r1", 0.46), uncited("r2", 0.52, sqls=[region_sql(), extra])))["share"] + assert share["classification"] == "sql_changed" and any("多了一次查询" in r for r in share["reasons"]) + + +def test_a_changed_default_model_is_a_template_change_for_agents_that_follow_it(): + base_graph = weekly_graph() + base_graph["defaults"] = {"provider": "anthropic", "model": "model-a"} + target_graph = copy.deepcopy(base_graph) + target_graph["defaults"]["model"] = "model-b" + result = run_compare(uncited("r1", 0.46, graph=base_graph), uncited("r2", 0.52, graph=target_graph)) + share = by_id(result)["share"] + assert share["classification"] == "sql_changed" and any("默认设置" in r for r in share["reasons"]), share["reasons"] + assert result["template"]["same_graph"] is False and compare.DEFAULTS_CHANGED in result["notes"] + # 取数节点不用模型：东区金额不受影响 + assert by_id(result)["east"]["classification"] == "unchanged" + # Agent 自己指定了模型：默认设置变了它不跟着变 + for graph in (base_graph, target_graph): + graph["nodes"][2]["data"]["config"]["model"] = "model-x" + share = by_id(run_compare(uncited("r1", 0.46, graph=base_graph), uncited("r2", 0.52, graph=target_graph)))["share"] + assert share["classification"] == "incomparable" + + +def cited(run_id, rows=BASE_ROWS, *, column="gmv", span=None): + """Agent 按出处交回数组字段：引用快照的哪一列、哪一段行由调用方定。""" + run = Fake(run_id) + run.tool("fetch", region_sql(), rows=rows) + bot = run.agent("bot", region_sql(), rows=rows, column=column, span=span) + first, last = span or (0, len(rows) - 1) + values = [r[COLUMNS.index(column)] for r in rows[first:last + 1]] + run.card([metric("total", "sum(vars.kpi.amounts)", sum(values), + [amounts_in(bot, rows, column=column, span=span)])]) + return run + + +def test_an_agent_citing_another_column_is_not_a_data_change(): + """SQL 一字不差、数据一模一样，Agent 这次引用的是退款列：取的不是同一个数。""" + total = by_id(run_compare(cited("r1"), cited("r2", column="refund")))["total"] + assert total["classification"] == "incomparable" and total["attribution"] is None + assert any("gmv" in r and "refund" in r for r in total["reasons"]), total["reasons"] + + +def test_an_agent_citing_other_rows_is_not_a_data_change(): + """Agent 这次只引用了后两行：东区的数还在快照里，不能把它算成「东区的数据没了」。""" + total = by_id(run_compare(cited("r1"), cited("r2", span=(1, 2))))["total"] + assert total["classification"] == "incomparable" and total["attribution"] is None + assert any("行" in r for r in total["reasons"]), total["reasons"] + # 两边引用的都是整份结果（这次多了一个区域）：照常是数据变化 + total = by_id(run_compare(cited("r1"), cited("r2", [*NEXT_ROWS, ["北区", 90.0, 0.0]])))["total"] + assert total["classification"] == "data_only" and total["attribution"]["rows"] + + +def test_an_unknown_writer_on_one_side_is_named_plainly(): + """一边记不出 vars.adj 是哪个节点写的：说清是哪一边没记录，不写「None」。""" + def make(run_id, writer, value): + run = Fake(run_id, graph=weekly_graph(between=(node("calc", "code", "调整", code="x", assign_to="adj"),))) + run.tool("fetch", region_sql()) + run.agent("bot", region_sql()) + run.code("calc", value, "adj") + item = scalar_in("vars.adj", value, writer, "code" if writer else "state") + run.card([metric("calc", "vars.adj * 2", value * 2, [item])]) + return run + calc = by_id(run_compare(make("r1", None, 10), make("r2", "calc", 12)))["calc"] + assert calc["classification"] == "incomparable" + assert not any("None" in r for r in calc["reasons"]) and any("基准运行" in r for r in calc["reasons"]) + + +def test_a_list_run_input_is_compared_by_its_content(): + """运行输入是列表：卡上只记了「[2 项]」，按入口节点的产出还原出完整值再比。""" + def make(run_id, targets): + run = Fake(run_id, inputs={"targets": targets}) + run.add("node.started", "start", {"node_type": "input"}) + run.add("node.finished", "start", {"artifact": run.put({"targets": targets})}) + item = {"path": "input.targets", "value": f"[{len(targets)} 项]", "node_id": "start", "via": "input", + "field": "targets", "status": "ok"} + run.card([metric("goal", "sum(input.targets)", float(sum(targets)), [item])]) + return run + goal = by_id(run_compare(make("r1", [100.0, 200.0]), make("r2", [100.0, 500.0])))["goal"] + assert goal["classification"] == "params_only" and any("targets" in r for r in goal["reasons"]), goal["reasons"] + goal = by_id(run_compare(make("r1", [100.0, 200.0]), make("r2", [100.0, 200.0])))["goal"] + assert goal["classification"] == "unchanged" + + def test_a_run_input_change_is_a_parameter_change(): def make(run_id, goal): run = Fake(run_id, inputs={"goal": goal}) @@ -534,6 +860,107 @@ def test_an_agent_asking_in_another_order_is_paired_by_sql(): assert pairs == [("params_only", by_day[:20]), ("params_only", by_region[:20])] +def test_swapped_queries_with_the_same_literals_pair_up_exactly(): + """Agent 两次都查了东区和南区，只是先后互换：先配字面量也相同的，两条都没变；下游的数照常是数据变化。""" + def region(name): + return f"SELECT SUM(amount) AS gmv FROM orders WHERE region = '{name}'" + + def make(run_id, regions, value, gmv): + run = Fake(run_id, graph=weekly_graph(between=(node("calc", "code", "调整", code="x", assign_to="adj"),))) + run.plain_agent("bot", [region(r) for r in regions], {"text": "ok"}, ["gmv"], [[gmv]], types={"gmv": "number"}) + run.code("calc", value, "adj") + run.card(code_metrics(value)(None, None, None, None)) + return run + + result = run_compare(make("r1", ["东区", "南区"], 10, 1.0), make("r2", ["南区", "东区"], 12, 2.0)) + assert [(q["structure"], q["literals"]) for q in result["queries"]] == [("same", []), ("same", [])] + assert by_id(result)["calc"]["classification"] == "data_only" + + +def test_a_switched_data_source_is_a_structure_change(): + """SQL 一字不差、换了数据源：查的不是同一份数据。""" + result = run_compare(week("r1"), week("r2", NEXT_ROWS, source="shop_backup")) + fetch = result_queries(result, "fetch") + assert fetch["structure"] == "changed" and any("shop_backup" in d for d in fetch["differences"]) + assert by_id(result)["east"]["classification"] == "sql_changed" + + +def test_a_literal_edited_in_the_workflow_is_a_template_change(): + """图里写死的筛选值改了（'paid' 改成 'refunded'）：执行的 SQL 只差字面量，但这是改模板，不是运行时代入的参数。""" + paid, refunded = region_sql(extra=" AND status = 'paid'"), region_sql(extra=" AND status = 'refunded'") + result = run_compare(week("r1", sql=paid, graph=weekly_graph(paid)), + week("r2", NEXT_ROWS, sql=refunded, graph=weekly_graph(refunded))) + east = by_id(result)["east"] + assert east["classification"] == "sql_changed", east["reasons"] + assert any("区域取数" in r and "'paid'" in r for r in east["reasons"]) + assert east["decomposition"] is None and east["attribution"] is None + # 查询表照实写：执行的 SQL 确实只差这一处字面量 + assert result_queries(result, "fetch")["structure"] == "params_only" + + +def test_reformatting_the_workflow_sql_is_not_a_change(): + loose = region_sql().replace(" FROM ", "\n FROM ").lower() + result = run_compare(week("r1"), week("r2", NEXT_ROWS, sql=loose, graph=weekly_graph(loose))) + assert {m["classification"] for m in result["metrics"]} == {"data_only"} + + +def test_renaming_a_node_is_not_a_change(): + graph = weekly_graph() + graph["nodes"][2]["data"]["label"] = "区域分析（新）" + result = run_compare(week("r1"), week("r2", NEXT_ROWS, graph=graph)) + assert {m["classification"] for m in result["metrics"]} == {"data_only"} + [renamed] = result["template"]["nodes"] + assert (renamed["node_id"], renamed["keys"]) == ("bot", ["label"]) + + +MERGE_SQL = "SELECT region, gmv FROM a ORDER BY region" +OUTER_SQL = "SELECT region, gmv FROM m ORDER BY region" +MERGED_TYPES = {"region": "text", "gmv": "number"} + + +def merged(run_id, sql, rows, *, template=None, nested=False, twice=False): + """取数节点查一次，合并节点把它当输入 a 再执行一条合并 SQL（nested：再套一层合并；twice：这次多接一个输入 b）， + 口径卡取合并结果的一格。""" + nodes = [node("start", "input", "开始"), node("fetch", "tool", "区域取数", tool=TOOL, args={"sql": template or sql}), + node("join", "merge", "合并", sql=MERGE_SQL, inputs={"a": "fetch", **({"b": "fetch"} if twice else {})})] + if nested: + nodes.append(node("outer", "merge", "再合并", sql=OUTER_SQL, inputs={"m": "join"})) + nodes.append(node("card", "metrics", "口径卡", caliber="周报口径", caliber_version="v1")) + run = Fake(run_id, graph=chain(*nodes)) + fetch = run.tool("fetch", sql, rows=rows) + table = [[r[0], r[1]] for r in rows] + inputs = [("a", "fetch", fetch)] + ([("b", "fetch", fetch)] if twice else []) + top, owner = run.merge("join", MERGE_SQL, inputs, ["region", "gmv"], table, types=MERGED_TYPES), "join" + if nested: + top, owner = run.merge("outer", OUTER_SQL, [("m", "join", top)], ["region", "gmv"], table, + types=MERGED_TYPES), "outer" + run.card([metric("east", f"cell(nodes.{owner}, 0, 'gmv')", table[0][1], + [cell_in(0, "gmv", table[0][1], top, node_id=owner)], name="东区金额")]) + return run + + +def test_a_merged_cell_follows_the_queries_feeding_the_merge(): + """合并 SQL 一字没改，喂它的取数 SQL 加了筛选：读合并结果的指标是 SQL 变了，不拆、不归因（嵌套的合并也追到底）。""" + paid = region_sql(extra=" AND status = 'paid'") + for nested in (False, True): + east = by_id(run_compare(merged("r1", region_sql(), BASE_ROWS, nested=nested), + merged("r2", paid, NEXT_ROWS, nested=nested)))["east"] + assert east["classification"] == "sql_changed", (nested, east["reasons"]) + assert east["decomposition"] is None and east["attribution"] is None + assert any("区域取数" in r and "SQL" in r for r in east["reasons"]), east["reasons"] + # 上游只是代进去的日期不同：参数变化 + template = region_sql(("{{ input.start }}", "{{ input.end }}")) + east = by_id(run_compare(merged("r1", region_sql(), BASE_ROWS, template=template), + merged("r2", region_sql(WEEK37), NEXT_ROWS, template=template)))["east"] + assert east["classification"] == "params_only", east["reasons"] + # 合并 SQL 和上游都没变：数据变化 + east = by_id(run_compare(merged("r1", region_sql(), BASE_ROWS), merged("r2", region_sql(), NEXT_ROWS)))["east"] + assert east["classification"] == "data_only", east["reasons"] + # 合并节点这次多接了一个输入 + east = by_id(run_compare(merged("r1", region_sql(), BASE_ROWS), merged("r2", region_sql(), NEXT_ROWS, twice=True))) + assert east["east"]["classification"] == "sql_changed" and any("「b」" in r for r in east["east"]["reasons"]) + + def test_an_agent_reconfigured_with_the_same_sql_is_a_template_change(): """Agent 改了提示词、SQL 一字不差：它按出处交来的字段可能换了取法，不能报成数据变化。""" graph = weekly_graph() From 3254744f6daf359418bf8d978a6c49000b17c542 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 11:28:41 +0800 Subject: [PATCH 7/8] =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=AF=B9=E6=AF=94?= =?UTF-8?q?=E5=A4=8D=E6=A0=B8=E4=BF=AE=E6=AD=A3=EF=BC=88=E7=AC=AC=202=20?= =?UTF-8?q?=E8=BD=AE=EF=BC=89=EF=BC=9A=E8=A1=8C=E6=AE=B5=E7=9B=B8=E5=90=8C?= =?UTF-8?q?=E3=80=81=E4=B8=80=E8=BE=B9=E6=98=AF=E6=95=B4=E4=BB=BD=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E7=9A=84=E5=BC=95=E7=94=A8=E4=B8=8D=E5=86=8D=E7=AE=97?= =?UTF-8?q?=E5=90=8C=E4=B8=80=E5=A4=84=EF=BC=8C=E9=BB=98=E8=AE=A4=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E7=9A=84=E5=8F=98=E5=8C=96=E5=8F=AA=E8=90=BD=E5=88=B0?= =?UTF-8?q?=E4=BC=9A=E8=AF=BB=E5=AE=83=E7=9A=84=E8=8A=82=E7=82=B9=E4=B8=8A?= =?UTF-8?q?=EF=BC=8C=E4=B8=8A=E6=B8=B8=20Agent=20=E4=BA=A4=E5=87=BA?= =?UTF-8?q?=E7=9A=84=E7=BB=93=E6=9E=9C=E4=B8=8D=E5=90=8C=E6=97=B6=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E8=AF=B4=E5=8F=98=E5=8C=96=E6=9D=A5=E8=87=AA=E4=B8=8A?= =?UTF-8?q?=E6=B8=B8=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Agent 两次引用同一段行，一边是整份结果、一边不是（这次多了北区，没引用的西区还在快照里）：判为无法比较，写明哪边引用了整份结果、另一边引用了哪几行、结果共几行；快照读不出来、核对不了是不是整份结果的同样不算同一处。以前按同一处算，西区被算成「数据没了」，占比超过 100% - 图级默认设置的变化按「哪类节点运行时会读这一项」落到节点上（_CFG_KEYS，照各执行器经 NodeContext.cfg 读的配置项登记）：结论句裁判、结论句的处理只有报告节点读，不再把 Agent 记成配置变了；条件分支让模型分类、结构校验让模型修复、知识检索用模型重排时也读默认模型，同样落到它们身上。新测试按执行器源码核对这张表，执行器新读了一项没登记就失败 - 代码、数据整形这类节点的上游有 Agent、模型调用、人工审批等节点，两次交出的产出不同（工件按内容寻址，按工件 id 比；产出没落成工件的核对不了）：它交的数变了不再说变化来自上游数据，判为无法比较并点名是哪个节点 - 上游节点改了配置时说明里点名是哪个节点；默认设置那句拆短，写出跟着变的是哪几项（模型接入不写 provider） - 补测试守住复核列出的未覆盖规则：容器运行输入还原不了、合并查询的输入追不下去、上游只有运行输入变了、审批默认设置、图里 SQL 只改格式（代码节点在下游）、跨节点先配字面量相同的、少了一次查询、只有一边落在快照上、查过库的写入节点改了配置、代码节点上游有节点改了配置、旧口径叶子的出处说不清时不拆、单格引用换了行；变异 99 处全部打红 --- backend/app/engine/compare.py | 183 +++++++++++-- backend/tests/fixtures/compare_runs.py | 5 +- backend/tests/test_compare_engine.py | 339 ++++++++++++++++++++++++- 3 files changed, 486 insertions(+), 41 deletions(-) diff --git a/backend/app/engine/compare.py b/backend/app/engine/compare.py index c899b533..b727210a 100644 --- a/backend/app/engine/compare.py +++ b/backend/app/engine/compare.py @@ -22,12 +22,13 @@ - cell() 那一格、Agent 按出处交来的字段落在哪份快照上就是哪条查询，两边各自的快照直接比 SQL；快照是合并查询的 结果时，再按别名一路追到喂它的每一份上游快照。只差字面量时还要看图里的 SQL：图里的 SQL 改过（改了写死的筛选值、 系数）是模板变化，图里一字未改的才是运行时代入的参数 - - Agent 按出处交来的字段还要引用同一列、同一段行（两边都引用整份结果的不算换了）：取哪一列、哪几行是 Agent 每次 - 运行时定的，换了就不是同一个数 + - Agent 按出处交来的字段还要引用同一列、同一段行（两边都引用整份结果的不算换了；行段相同、一边是整份结果一边 + 不是的算换了）：取哪一列、哪几行是 Agent 每次运行时定的，换了就不是同一个数 - 不落在快照上的：模型、人交出的值（Agent 没按出处交、模型调用、人工审批……）没有出处，值变了说不清是数据还是它 - 本身的差异；代码、数据整形这类节点看它自己查过的库，不查库的看它的配置和画布上的全部上游，上游的查询结果两次 - 一模一样、值却变了的同样说不清 - - 图级的变化落到节点上：连线有增删的，终点节点算改了；默认设置改了的，没有自行设置这几项的模型类节点算改了 + 本身的差异；代码、数据整形这类节点看它自己查过的库，不查库的看它的配置和画布上的全部上游。上游的查询结果两次 + 一模一样、值却变了的同样说不清；上游有模型、人交出值的节点、两次交出的不同的也说不清 + - 图级的变化落到节点上：连线有增删的，终点节点算改了；默认设置改了的，运行时会读这几项（_CFG_KEYS）、又没有自行 + 设置的节点算改了 - 严格拆分（只对口径变了的）：旧表达式的每个叶子路径都能在这次的口径卡输入里取得值时， 口径效应 = f_new(x_new) − f_old(x_new)，数据效应 = f_old(x_new) − f_old(x_old)。f_old(x_new) 用推演重算 （engine/recompute.py）的还原和求值算，取整按旧口径卡的小数位；三个数按十进制精确相减，两效应之和逐位等于 @@ -88,12 +89,49 @@ _RATIO = "percent_of_ratio" #: 图里写 SQL 的配置项：调用工具节点的参数、合并查询的合并 SQL _SQL_KEY = {"tool": "args.sql", "merge": "sql"} #: 交出的值由模型或人给出的节点（类型、口径卡记的来历）：同样的输入、同样的配置，交出的值也可能不同 -_GENERATED = frozenset({"agent", "llm", "supervisor", "human", "memory", "retrieve", "subgraph"}) -_GENERATED_VIA = frozenset({"agent", "agent_field", "llm", "team", "human", "memory", "retrieval", "subgraph"}) -#: 图级默认设置只有模型类节点在读（节点没配 provider、model 时回退到它，GraphSpec.defaults） -_MODEL_TYPES = frozenset({"llm", "agent", "supervisor"}) +_GENERATED = frozenset({"agent", "llm", "supervisor", "human", "memory", "retrieve", "subgraph", "report"}) +_GENERATED_VIA = frozenset({"agent", "agent_field", "llm", "team", "human", "memory", "retrieval", "subgraph", + "report"}) +#: 调模型时读的配置项（nodes/llm._model_spec） +_MODEL_KEYS = frozenset({"provider", "model", "temperature", "max_tokens", "thinking", "effort"}) +#: 各类节点运行时按 NodeContext.cfg 读的配置项：节点上没写这一项就回退到图级默认设置（GraphSpec.defaults），默认 +#: 设置改了这一项，没自行设置它的这类节点交出的值可能跟着变；不读它的不受影响（结论句裁判只有报告节点读）。读了也 +#: 不影响交出的值的见 _QUIET_DEFAULTS。合并查询、双盲复算、反方审稿不经 cfg 读配置。执行器新读了一项要在这里登记， +#: tests/test_compare_engine.py 按源码核对 +_CFG_KEYS: dict[str, frozenset[str]] = { + "input": frozenset({"fields"}), + "output": frozenset({"fields", "contract"}), + "transform": frozenset({"mode", "expression", "template", "assign_to"}), + "llm": _MODEL_KEYS | {"system", "skills", "use_history", "prompt", "output_schema", "emit_message", "assign_to"}, + "agent": _MODEL_KEYS | {"system", "skills", "use_history", "prompt", "output_schema", "cite_fields", "tools", + "memory_scope", "collection", "parallel_tools", "max_steps", "budget_tokens", + "budget_usd", "approval", "emit_message", "assign_to"}, + "supervisor": frozenset({"agents", "max_rounds", "on_exhausted", "goal", "provider", "model", "max_tokens", + "max_parallel", "approval", "emit_message", "assign_to"}), + "subgraph": frozenset({"workflow_id", "workflow_version", "input", "assign_to"}), + "tool": frozenset({"tool", "args", "fail_fast", "memory_scope", "collection", "approval", "assign_to"}), + "code": frozenset({"code", "evidence_role", "language", "timeout", "memory_mb", "network", "isolation", "files", + "fail_fast", "approval", "assign_to"}), + "branch": frozenset({"cases", "mode"}), + "loop": frozenset({"mode", "max_iterations", "item_var", "items", "condition"}), + "human": frozenset({"mode", "title", "message", "schema", "context", "draft", "stop_on_reject", "assign_to"}), + "validate": frozenset({"schema", "source", "max_retries", "repair_with_llm", "fail_fast", "assign_to"}), + "memory": frozenset({"action", "scope", "query", "limit", "kind", "content", "importance", "assign_to"}), + "retrieve": frozenset({"query", "collection", "limit", "alpha", "rerank", "min_score", "assign_to"}), + "metrics": frozenset({"metrics", "caliber", "caliber_version", "on_missing", "assign_to"}), + "report": _MODEL_KEYS | {"numbers", "claims", "judge", "entities", "max_repairs", "system", "instructions", + "on_violation", "evidence_from", "metrics_from", "emit_message", "assign_to"}, +} +#: 只在调模型时才读的（_calls_model）：条件分支让模型分类、结构校验让模型修复（默认开着）、知识检索用模型重排 +_CFG_KEYS_IF: dict[str, frozenset[str]] = { + "branch": frozenset({"input", "instruction", "provider", "model"}), + "validate": frozenset({"provider", "model"}), + "retrieve": _MODEL_KEYS, +} #: 改了也不影响交出的值的默认设置：审批策略只管调用工具前问不问人 _QUIET_DEFAULTS = frozenset({"approval"}) +#: 配置键的叫法，engine/labels 里没有的这两个：节点类型；模型服务叫「模型接入」（lib/terms.ts），不写 provider +_KEY_LABEL = {"type": "节点类型", "provider": "模型接入"} #: 浮点能原样写回的十进制有效数字位数（DBL_DIG） _FLOAT_DIGITS = 15 @@ -267,6 +305,16 @@ def _queries(side: Side) -> list[_Query]: return out +def _outputs(side: Side) -> dict[str, tuple[Any, ...]]: + """每个节点在封存范围内交出的产出工件（node.finished 的 artifact），按执行的先后。工件按内容寻址：两次运行的同一个 + 节点交出的一模一样，工件 id 就一样。产出没落成工件的记 None，核对不了。""" + out: dict[str, list[Any]] = {} + for _, etype, node_id, data in side.events: + if etype == EventType.NODE_FINISHED and node_id and isinstance(data, dict): + out.setdefault(node_id, []).append(data.get("artifact")) + return {node_id: tuple(found) for node_id, found in out.items()} + + def _pair(base: list[_Query], target: list[_Query], norm: Callable[[_Query], NormalizedSql | None]) -> list[tuple[_Query | None, _Query | None]]: """配对，一轮比一轮宽： @@ -563,7 +611,7 @@ def _config_keys(was: dict[str, Any], now: dict[str, Any]) -> tuple[list[str], b def _key_label(key: str, node_type: Any) -> str: - return "节点类型" if key == "type" else field_label(key, node_type) + return _KEY_LABEL.get(key) or field_label(key, node_type) def _unset(value: Any) -> bool: @@ -571,6 +619,29 @@ def _unset(value: Any) -> bool: return value in (None, "") +def _reads(node: dict[str, Any], defaults: dict[str, Any]) -> frozenset[str]: + """这个节点运行时按 NodeContext.cfg 读的配置项（_CFG_KEYS）。条件分支、结构校验、知识检索调不调模型看它那次的 + 设置，调模型的才读模型那几项。""" + kind = str(node.get("type")) + keys = _CFG_KEYS.get(kind, frozenset()) + if kind in _CFG_KEYS_IF and _calls_model(kind, node, defaults): + keys = keys | _CFG_KEYS_IF[kind] + return keys + + +def _calls_model(kind: str, node: dict[str, Any], defaults: dict[str, Any]) -> bool: + """条件分支是让模型分类、结构校验让模型修复（默认开着）、知识检索用模型重排：和执行器同一个判断、同一个缺省。""" + def cfg(key: str, default: Any) -> Any: + value = _config(node).get(key) + return defaults.get(key, default) if _unset(value) else value + + if kind == "branch": + return cfg("mode", "expression") == "llm" + if kind == "validate": + return bool(cfg("repair_with_llm", True)) + return (cfg("rerank", "off") or "off") == "model" + + def _graph_sql(node: dict[str, Any] | None) -> tuple[str, str | None] | None: """节点在图里写的 SQL 和它的数据源：调用工具节点是参数里的 SQL（数据源从工具名来），合并查询是合并 SQL。 别的节点（Agent 自己写 SQL）返回 None。""" @@ -641,9 +712,12 @@ class _Compare: self.base_nodes, self.target_nodes = _nodes(base.graph), _nodes(target.graph) self.base_queries, self.target_queries = _queries(base), _queries(target) self.pairs = _pair(self.base_queries, self.target_queries, self._norm_of) - self.template, self.changed = self._template() + self.template, self.changed, self.inherited = self._template() self.titles = {**{k: self._title_of(n) for k, n in self.base_nodes.items()}, **{k: self._title_of(n) for k, n in self.target_nodes.items()}} + #: 画布上的先后（这次运行的图在前）：说明里点名上游节点时按它挑第一个 + self.order = {n: i for i, n in enumerate(dict.fromkeys([*self.target_nodes, *self.base_nodes]))} + self.base_outputs, self.target_outputs = _outputs(base), _outputs(target) self.parents: dict[str, set[str]] = {} for source, target_id, _ in _edges(base.graph) | _edges(target.graph): self.parents.setdefault(target_id, set()).add(source) @@ -865,9 +939,25 @@ class _Compare: def _moved(self, node_id: str) -> str: """一个影响取值的节点变化（self.changed）写成一句话。""" name = f"上游节点「{self.title(node_id)}」" - return {"added": f"{name}是这次新增的", "removed": f"{name}这次被删除了", "edges": f"{name}的连线有变化", - "defaults": f"工作流默认设置有变化，{name}没有单独设置这些项，随之改变" - }.get(self.changed.get(node_id, ""), f"{name}的配置有变化") + change = self.changed.get(node_id, "") + if change == "defaults": + keys = self.inherited.get(node_id, []) + shown = "、".join(_key_label(k, self._type_of(node_id)) for k in keys[:3]) + more = f"等 {len(keys)} 项" if len(keys) > 3 else "" + return f"{name}跟随的工作流默认设置（{shown}{more}）有变化" + return {"added": f"{name}是这次新增的", "removed": f"{name}这次被删除了", + "edges": f"{name}的连线有变化"}.get(change, f"{name}的配置有变化") + + def _ordered(self, nodes: Iterable[str]) -> list[str]: + """按画布上的先后排（这次运行的图在前）。""" + return sorted(nodes, key=lambda n: self.order.get(n, len(self.order))) + + def _regenerated(self, node_id: str) -> bool: + """模型、人交出值的节点（_GENERATED）两次交出的产出不同，或者核对不了。一模一样的没带进它本身的差异。""" + if self._type_of(node_id) not in _GENERATED: + return False + before, after = self.base_outputs.get(node_id, ()), self.target_outputs.get(node_id, ()) + return before != after or None in before or None in after def provenance(self, path: str, b: Any, t: Any, *, moved: bool = True) -> _Change: """一个输入的出处在两次运行之间变没变（b、t 是推演重算还原出来的输入）。moved 是这个指标的值变没变：输入是 @@ -904,11 +994,19 @@ class _Compare: return _Change("template", self._moved(writer)) # 写它的节点不查库（代码、数据整形……）：看画布上它的全部上游；写它的节点不知道就看整张图 scope = self._upstream(writer) if writer else set(self.titles) - moved_nodes = [n for n in scope if n in self.changed] + moved_nodes = self._ordered(n for n in scope if n in self.changed) worst = _worst(self._query_change(b2, t2) for b2, t2 in self.pairs if (t2 or b2).node_id in scope) where = f"来自「{self.title(writer)}」" if writer else "不知道由哪个节点写入" - if moved_nodes or worst.kind in ("sql", "template", "unknown"): + if moved_nodes: + return _Change("unknown", f"输入「{path}」{where}，{self._moved(moved_nodes[0])}，无法判断变化来源") + if worst.kind in ("sql", "template", "unknown"): return _Change("unknown", f"输入「{path}」{where}，对不上具体的查询；它的上游有变化，无法判断变化来源") + if not same and (spoken := self._ordered(n for n in scope if self._regenerated(n))): + # 上游有模型、人交出的值，两次不同：它交的数变了，可能是数据变了，也可能是模型本身的差异（它可能用的 + # 正是 Agent 没按出处交的数）。上游的查询结果变没变都一样说不清 + kind = TYPE_LABEL.get(self._type_of(spoken[0]) or "", "节点") + return _Change("unknown", f"输入「{path}」{where}，它的上游节点「{self.title(spoken[0])}」（{kind}）两次" + "交出的结果不同，无法判断变化是否来自数据") if worst.kind == "same" and not same and not self._data_moved(scope): # 上游查到的数据两次一模一样，它交的值却变了：不是数据变的。上游只有运行输入变了的，算参数 if (self.base.inputs or {}) != (self.target.inputs or {}) \ @@ -939,16 +1037,35 @@ class _Compare: return change def _citation(self, agent: Any, b: Any, t: Any, qb: _Query, qt: _Query) -> _Change | None: - """Agent 按出处交来的字段两次引用的是不是同一处：同一列（对象数组是同一组列）、同一行或同一段行。行段不同、但 - 两边引用的都是整份结果（这次多了几行）的算同一处。不是同一处返回说明。""" + """Agent 按出处交来的字段两次引用的是不是同一处：同一列（对象数组是同一组列）、同一行或同一段行。两边引用的都是 + 整份结果的算同一处（这次多了几行也是）；行段相同、一边是整份结果一边不是的不算——没引用的那几行还在这次的快照 + 里，按维度配对会把它们算成「数据没了」。不是同一处、核对不了是不是同一处的，返回说明。""" name = f"Agent「{self.title(agent)}」" if agent else "Agent" lb, lt = _locator(b), _locator(t) cb, ct = lb.get("column") or lb.get("columns"), lt.get("column") or lt.get("columns") if cb != ct: return _Change("unknown", f"{name}引用的列由{_columns_text(cb)}改为{_columns_text(ct)}，取的不是同一个数") - if (b.row, b.rows) == (t.row, t.rows) or (b.row is None and t.row is None and _whole(b, qb) and _whole(t, qt)): + what = "同一个数" if b.row is not None and t.row is not None else "同一组数" + changed = _Change("unknown", f"{name}引用的行由{_rows_text(b)}改为{_rows_text(t)}，取的不是{what}") + if b.row is not None or t.row is not None: + # 单格（数组字段里的一个元素、按出处交的一格）：取哪一行由口径卡的下标和 Agent 交的行段一起定，同一行才是 + # 同一处 + return None if (b.row, b.rows) == (t.row, t.rows) else changed + wb, wt = _whole(b, qb), _whole(t, qt) + same_span = b.rows == t.rows + if (wb and wt) or (same_span and wb is False and wt is False): return None - return _Change("unknown", f"{name}引用的行由{_rows_text(b)}改为{_rows_text(t)}，取的不是同一组数") + if (wb is None or wt is None) and (same_span or (wb is not False and wt is not False)): + # 快照读不出来，核对不了是不是整份结果：行段相同的可能一边整份一边不是，行段不同的也可能两边都是整份 + side = "基准运行" if wb is None else "这次运行" + return _Change("unknown", f"{side}的查询快照无法读取，无法核对 {name}引用的是不是整份结果") + if not same_span: + return changed + if wb: + return _Change("unknown", f"{name}在基准运行中引用了整份结果，这次只引用了{_rows_text(t)}" + f"（结果共 {len(qt.snap['rows'])} 行），取的不是同一组数") + return _Change("unknown", f"{name}在基准运行中只引用了{_rows_text(b)}（结果共 {len(qb.snap['rows'])} 行），" + "这次引用了整份结果，取的不是同一组数") def _generated(self, path: str, writer: str, same: bool) -> _Change: """模型、人交出的值（Agent 没按出处交、模型调用、人工审批……）：没有出处，值变了说不清是数据变了还是它本身的 @@ -982,15 +1099,17 @@ class _Compare: # ---------------------------------------------------------------- 模板 - def _template(self) -> tuple[dict[str, Any], dict[str, str]]: + def _template(self) -> tuple[dict[str, Any], dict[str, str], dict[str, list[str]]]: """两次运行的图快照比较：新增、删除的节点，配置有变化的节点和配置项名（不展示值）。 changed 是影响取值的变化，判断输入的出处时用：节点 → added / removed / config / edges / defaults。只改名称的 不算；图里的 SQL 只改了空白、大小写的不算（规范化以后一模一样）。图级的变化也落到节点上：连线有增删的，终点 - 节点交出的值可能变了（接进来的东西、执行的条件变了）；默认设置改了的，没有自行设置这几项的模型类节点跟着变。""" + 节点交出的值可能变了（接进来的东西、执行的条件变了）；默认设置改了的，运行时会读这几项（_reads）、又没有自行 + 设置的节点跟着变，inherited 记着各自跟着变的是哪几项。""" before, after = self.base_nodes, self.target_nodes nodes: list[dict[str, Any]] = [] changed: dict[str, str] = {} + inherited: dict[str, list[str]] = {} def entry(node_id: str, node: dict[str, Any], change: str, keys: list[str]) -> dict[str, Any]: return {"node_id": node_id, "title": self._title_of(node), "change": change, "keys": keys, @@ -1023,9 +1142,14 @@ class _Compare: if defaults_before.get(k) != defaults_after.get(k) and k not in _QUIET_DEFAULTS] for node_id, node in after.items(): was = before.get(node_id) - if differing and was is not None and str(node.get("type")) in _MODEL_TYPES and any( - _unset(_config(node).get(k)) and _unset(_config(was).get(k)) for k in differing): + if not differing or was is None: + continue + reads = _reads(was, defaults_before) | _reads(node, defaults_after) + hit = [k for k in differing + if k in reads and _unset(_config(node).get(k)) and _unset(_config(was).get(k))] + if hit: changed.setdefault(node_id, "defaults") + inherited[node_id] = hit same_edges = edges_before == edges_after same_defaults = defaults_before == defaults_after if not same_edges: @@ -1033,7 +1157,7 @@ class _Compare: if not same_defaults: self.notes.append(DEFAULTS_CHANGED) return {"base_version": self.base.run.get("version"), "target_version": self.target.run.get("version"), - "same_graph": not nodes and same_edges and same_defaults, "nodes": nodes}, changed + "same_graph": not nodes and same_edges and same_defaults, "nodes": nodes}, changed, inherited # ---------------------------------------------------------------- 指标 @@ -1422,11 +1546,14 @@ def _same_input(b: Any, t: Any) -> bool | None: return recompute._same(b.item.get("value"), t.item.get("value")) -def _whole(inp: Any, q: _Query) -> bool: - """这个输入引用的是整份查询结果：数组字段的行段从第一行到最后一行，或者容器本身就是整份结果。""" +def _whole(inp: Any, q: _Query) -> bool | None: + """这个输入引用的是不是整份查询结果：容器本身就是整份结果，或者数组字段的行段从第一行到最后一行。快照读不出来、 + 不知道结果共几行的返回 None。""" if inp.table: return True - return inp.rows is not None and q.snap is not None and inp.rows == (0, len(q.snap["rows"]) - 1) + if q.snap is None: + return None + return inp.rows is not None and inp.rows == (0, len(q.snap["rows"]) - 1) def _rows_text(inp: Any) -> str: diff --git a/backend/tests/fixtures/compare_runs.py b/backend/tests/fixtures/compare_runs.py index a961018e..9e0d3944 100644 --- a/backend/tests/fixtures/compare_runs.py +++ b/backend/tests/fixtures/compare_runs.py @@ -38,10 +38,13 @@ def chain(*nodes): return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} -def weekly_graph(sql=None, *, between=(), card_label="周报口径卡"): +def weekly_graph(sql=None, *, between=(), after_fetch=(), card_label="周报口径卡"): + """一条链：开始 → 取数 → [after_fetch] → Agent → [between] → 口径卡 → 写周报 → 成果。after_fetch 里的节点不在 + Agent 的下游（它的上游只有取数节点），between 里的在。""" return chain( node("start", "input", "开始"), node("fetch", "tool", "区域取数", tool=TOOL, args={"sql": sql or region_sql()}), + *after_fetch, node("bot", "agent", "区域分析", prompt="按区域列出金额", tools=[TOOL], cite_fields=True), *between, node("card", "metrics", card_label, caliber="周报口径", caliber_version="v1"), diff --git a/backend/tests/test_compare_engine.py b/backend/tests/test_compare_engine.py index 328b0044..374ad4ae 100644 --- a/backend/tests/test_compare_engine.py +++ b/backend/tests/test_compare_engine.py @@ -9,16 +9,20 @@ SQL，Agent 节点 bot 自己再查一次、把各区域的金额作为数组字 为 1；对不上时占比为空并说明；截断的快照不归因，照实写 - 遮罩列的值绝不出现在结果里；用到遮罩列的指标不拆分 - 模板对比：新增、删除的节点，配置有变化的节点和配置项名 -- 输入的出处：合并查询追到它的上游查询；Agent 按出处交的字段还要引用同一列、同一段行；没按出处交的数、上游数据 - 没变数却变了的，说不清变化从哪来（incomparable）；图里写死的字面量改了、默认设置和连线变了都算模板变化 +- 输入的出处：合并查询追到它的上游查询；Agent 按出处交的字段还要引用同一列、同一段行（行段相同、一边是整份结果 + 一边不是的不算）；没按出处交的数、上游数据没变数却变了的、上游的 Agent 两次交出的结果不同的，说不清变化从哪来 + （incomparable）；图里写死的字面量改了、连线变了、读到的默认设置变了都算模板变化 """ from __future__ import annotations import copy +import inspect import json +import re from decimal import Decimal from app.engine import compare +from app.engine.compiler import RUNNERS from app.engine.merge_query import MERGE_SOURCE from tests.fixtures.compare_runs import ( BASE_ROWS, @@ -70,7 +74,9 @@ def test_same_sql_new_rows_is_data_only(): assert (east["base_rendered"], east["target_rendered"], east["delta_rendered"]) == ("1,200.5元", "1,500.5元", "+300.0元") assert east["delta_pct"] == round(300 / 1200.5 * 100, 2) assert east["card_id"] == "card" and east["card_title"] == "周报口径卡" and east["name"] == "东区金额" - assert east["decomposition"] is None and east["decomposition_note"] is None and east["reasons"] + assert east["decomposition"] is None and east["decomposition_note"] is None + # 两个输入都落在查询快照上：直接说取数的 SQL 没变、查询结果变了 + assert east["reasons"] == total["reasons"] == [compare.DATA_FROM_QUERIES] assert total["delta"] == 200.0 and total["delta_rendered"] == "+200.0元" assert result["summary"] == {"unchanged": 0, "data_only": 2, "caliber_and_data": 0, "sql_changed": 0, "params_only": 0, "added": 0, "removed": 0, "incomparable": 0} @@ -558,10 +564,14 @@ def code_metrics(value): return build +def calc_node(code="print(1)"): + return node("calc", "code", "调整", code=code, assign_to="adj") + + def test_an_input_from_a_code_node_follows_its_upstream(): + """代码节点接在取数节点后面（不在 Agent 下游）：看它画布上的全部上游。""" def make(run_id, rows, value, sql=None): - run = Fake(run_id, graph=weekly_graph(sql, between=(node("calc", "code", "调整", code="print(1)", - assign_to="adj"),))) + run = Fake(run_id, graph=weekly_graph(sql, after_fetch=(calc_node(),))) sql = sql or region_sql() run.tool("fetch", sql, rows=rows) run.agent("bot", sql, rows=rows) @@ -583,11 +593,73 @@ def test_an_input_from_a_code_node_follows_its_upstream(): assert compare.DATA_FROM_UPSTREAM not in calc["reasons"] +def test_a_code_node_after_an_agent_that_answered_differently_is_incomparable(): + """代码节点在 Agent 下游，算的是 Agent 没按出处交的占比：Agent 两次交出的结果不同，变化可能来自数据，也可能来自 + 模型本身的差异，不能说变化来自上游数据。Agent 两次交出的一模一样时照常（见 + test_swapped_queries_with_the_same_literals_pair_up_exactly）。""" + def make(run_id, rows, share, value=None): + value = share * 2 if value is None else value + run = Fake(run_id, graph=weekly_graph(between=(calc_node("result = nodes.bot.data.kpi.share * 2"),))) + run.tool("fetch", region_sql(), rows=rows) + run.plain_agent("bot", [region_sql()], {"kpi": {"share": share}}, rows=rows) + run.code("calc", value, "adj") + run.card(code_metrics(value)(None, None, rows, rows)) + return run + + calc = by_id(run_compare(make("r1", BASE_ROWS, 0.46), make("r2", NEXT_ROWS, 0.52)))["calc"] + assert calc["classification"] == "incomparable" and calc["attribution"] is None + assert calc["reasons"] == ["输入「vars.adj」来自「调整」，它的上游节点「区域分析」（Agent）两次交出的结果不同，" + "无法判断变化是否来自数据"] + + # Agent 的产出两次都没落成工件（写工件失败）：一样不一样核对不了，同样不报成数据变化 + def unstored(run): + run.events = [(seq, etype, nid, {k: v for k, v in data.items() if k != "artifact"} + if (etype, nid) == ("node.finished", "bot") else data) for seq, etype, nid, data in run.events] + return run + calc = by_id(run_compare(unstored(make("r1", BASE_ROWS, 0.46)), + unstored(make("r2", NEXT_ROWS, 0.46, value=1.0))))["calc"] + assert calc["classification"] == "incomparable" and "区域分析" in calc["reasons"][0] + + +def test_a_reconfigured_node_upstream_of_a_code_node_is_named(): + """代码节点上游的数据整形节点改了表达式（SQL 一字没改）：代码节点交的数变了说不清从哪来，说明里点出改的是哪个节点。""" + def make(run_id, expression, rows, value): + shape = node("shape", "transform", "整理", mode="expression", expression=expression, assign_to="kept") + run = Fake(run_id, graph=weekly_graph(after_fetch=(shape, calc_node()))) + run.tool("fetch", region_sql(), rows=rows) + run.agent("bot", region_sql(), rows=rows) + run.code("calc", value, "adj") + run.card(code_metrics(value)(None, None, rows, rows)) + return run + + calc = by_id(run_compare(make("r1", "nodes.fetch.rows", BASE_ROWS, 10), + make("r2", "nodes.fetch.rows[:2]", NEXT_ROWS, 12)))["calc"] + assert calc["classification"] == "incomparable" + assert calc["reasons"] == ["输入「vars.adj」来自「调整」，上游节点「整理」的配置有变化，无法判断变化来源"] + + +def test_a_run_input_upstream_of_a_code_node_is_a_parameter_change(): + """代码节点上游查到的数据两次一模一样，只有运行输入变了：它交的数变了算参数变化。""" + def make(run_id, goal, value): + run = Fake(run_id, graph=weekly_graph(after_fetch=(calc_node(),)), inputs={"goal": goal}) + run.add("node.started", "start", {"node_type": "input"}) + run.add("node.finished", "start", {"artifact": run.put({"goal": goal})}) + run.tool("fetch", region_sql()) + run.agent("bot", region_sql()) + run.code("calc", value, "adj") + run.card(code_metrics(value)(None, None, BASE_ROWS, BASE_ROWS)) + return run + + calc = by_id(run_compare(make("r1", 1000, 10), make("r2", 1500, 15)))["calc"] + assert calc["classification"] == "params_only" + assert calc["reasons"] == ["输入「vars.adj」来自「调整」，它的上游只有运行输入有变化"] + + def test_a_value_from_a_querying_node_that_moved_without_its_data_is_incomparable(): """调用工具节点整份交出的结果（不是 cell() 取的格）：它查到的行两次一样，交出的值（这里是查询耗时）却变了， 变化不来自数据；查到的行也变了的，照常是数据变化。""" - def make(run_id, elapsed, rows=BASE_ROWS): - run = Fake(run_id) + def make(run_id, elapsed, rows=BASE_ROWS, graph=None): + run = Fake(run_id, graph=graph) run.tool("fetch", region_sql(), rows=rows) run.agent("bot", region_sql(), rows=rows) run.card([metric("cost", "vars.res.elapsed_ms", elapsed, @@ -597,6 +669,12 @@ def test_a_value_from_a_querying_node_that_moved_without_its_data_is_incomparabl assert cost["classification"] == "incomparable" and any("区域取数" in r for r in cost["reasons"]) cost = by_id(run_compare(make("r1", 3), make("r2", 5, NEXT_ROWS)))["cost"] assert cost["classification"] == "data_only" + # 取数节点自己改了配置（SQL 没动）：它整份交出的结果可能跟着变，算模板变化 + graph = weekly_graph() + graph["nodes"][1]["data"]["config"]["fail_fast"] = False + cost = by_id(run_compare(make("r1", 3), make("r2", 5, NEXT_ROWS, graph)))["cost"] + assert cost["classification"] == "sql_changed", cost["reasons"] + assert cost["reasons"] == ["上游节点「区域取数」的配置有变化"] def test_data_from_upstream_is_only_claimed_when_the_graph_is_the_same(): @@ -609,7 +687,7 @@ def test_data_from_upstream_is_only_claimed_when_the_graph_is_the_same(): run.card(code_metrics(value)(None, None, rows, rows)) return run - base_graph = weekly_graph(between=(node("calc", "code", "调整", code="print(1)", assign_to="adj"),)) + base_graph = weekly_graph(after_fetch=(calc_node(),)) target_graph = copy.deepcopy(base_graph) target_graph["nodes"][-1]["data"]["config"] = {"fields": [{"name": "周报", "value": "{{ nodes.write.text }}"}]} calc = by_id(run_compare(make("r1", BASE_ROWS, 10, base_graph), make("r2", NEXT_ROWS, 12, target_graph)))["calc"] @@ -676,6 +754,27 @@ def test_an_extra_query_by_the_writer_is_a_sql_change(): extra = "SELECT COUNT(*) AS n FROM refunds" share = by_id(run_compare(uncited("r1", 0.46), uncited("r2", 0.52, sqls=[region_sql(), extra])))["share"] assert share["classification"] == "sql_changed" and any("多了一次查询" in r for r in share["reasons"]) + # 反过来这次少查了一次：同样是取数变了 + share = by_id(run_compare(uncited("r1", 0.46, sqls=[region_sql(), extra]), uncited("r2", 0.52)))["share"] + assert share["classification"] == "sql_changed" and any("少了一次查询" in r for r in share["reasons"]) + + +def test_an_input_on_a_snapshot_on_one_side_only_is_incomparable(): + """基准运行没找到东区金额那一格落在哪份快照上（卡上只记了值），这次找到了：只有一边对得上查询快照，比不了取数。""" + def make(run_id, rows, *, located): + run = Fake(run_id) + fetch = run.tool("fetch", region_sql(), rows=rows) + run.agent("bot", region_sql(), rows=rows) + item = cell_in(0, "gmv", rows[0][1], fetch) + if not located: + kept = {k: v for k, v in item.items() if k not in ("artifact", "locator", "eid")} + item = {**kept, "status": "unresolved"} + run.card([metric("east", EAST, rows[0][1], [item], name="东区金额")]) + return run + + east = by_id(run_compare(make("r1", BASE_ROWS, located=False), make("r2", NEXT_ROWS, located=True)))["east"] + assert east["classification"] == "incomparable" and east["attribution"] is None + assert east["reasons"] == [f"输入「{EAST}」只有一边对得上查询快照，无法比较取数"] def test_a_changed_default_model_is_a_template_change_for_agents_that_follow_it(): @@ -685,10 +784,20 @@ def test_a_changed_default_model_is_a_template_change_for_agents_that_follow_it( target_graph["defaults"]["model"] = "model-b" result = run_compare(uncited("r1", 0.46, graph=base_graph), uncited("r2", 0.52, graph=target_graph)) share = by_id(result)["share"] - assert share["classification"] == "sql_changed" and any("默认设置" in r for r in share["reasons"]), share["reasons"] + assert share["classification"] == "sql_changed" + assert share["reasons"] == ["上游节点「区域分析」跟随的工作流默认设置（模型）有变化"] assert result["template"]["same_graph"] is False and compare.DEFAULTS_CHANGED in result["notes"] # 取数节点不用模型：东区金额不受影响 assert by_id(result)["east"]["classification"] == "unchanged" + # 按出处交的字段：取法可能跟着模型变了 + total = by_id(run_compare(week("r1", graph=base_graph), week("r2", NEXT_ROWS, graph=target_graph)))["total"] + assert total["classification"] == "sql_changed" + assert total["reasons"] == ["上游节点「区域分析」跟随的工作流默认设置（模型）有变化，它按出处交来的字段可能换了取法"] + # 一次改了好几项：写界面上的叫法（模型接入不写 provider），多的只写项数 + many = copy.deepcopy(base_graph) + many["defaults"] = {"provider": "openai", "model": "model-b", "temperature": 0.5, "max_tokens": 2000} + share = by_id(run_compare(uncited("r1", 0.46, graph=base_graph), uncited("r2", 0.52, graph=many)))["share"] + assert share["reasons"] == ["上游节点「区域分析」跟随的工作流默认设置（模型接入、模型、温度等 4 项）有变化"] # Agent 自己指定了模型：默认设置变了它不跟着变 for graph in (base_graph, target_graph): graph["nodes"][2]["data"]["config"]["model"] = "model-x" @@ -696,6 +805,68 @@ def test_a_changed_default_model_is_a_template_change_for_agents_that_follow_it( assert share["classification"] == "incomparable" +def test_defaults_that_agents_do_not_read_leave_them_alone(): + """只改了 Agent 不读的默认设置：结论句裁判、结论句的处理只有报告节点读，审批策略只管调用工具前问不问人。按出处 + 交的字段、取数节点的格子照常是数据变化。""" + for before, after in (({"judge": {"max_cost_usd": 0.5}}, {"judge": {"max_cost_usd": 1.0}}), + ({"claims": "off"}, {"claims": "judge"}), + ({"approval": "dangerous"}, {"approval": "never"})): + base_graph, target_graph = weekly_graph(), weekly_graph() + base_graph["defaults"], target_graph["defaults"] = before, after + result = run_compare(week("r1", graph=base_graph), week("r2", NEXT_ROWS, graph=target_graph)) + found = {m["metric_id"]: (m["classification"], m["reasons"]) for m in result["metrics"]} + assert found == {"east": ("data_only", [compare.DATA_FROM_QUERIES]), + "total": ("data_only", [compare.DATA_FROM_QUERIES])}, (before, found) + assert compare.DEFAULTS_CHANGED in result["notes"] + + +def test_a_model_default_reaches_every_node_that_calls_the_model(): + """默认模型换了，跟着换的不只 Agent：让模型分类的条件分支、让模型修复的结构校验（默认开着）、用模型重排的知识检索 + 也是。它们在代码节点上游时，代码节点交的数变了不能说来自上游数据；不调模型的照常。""" + def calc(extra): + def make(run_id, model, rows, value): + graph = weekly_graph(after_fetch=(extra, calc_node())) + graph["defaults"] = {"provider": "anthropic", "model": model} + run = Fake(run_id, graph=graph) + run.tool("fetch", region_sql(), rows=rows) + run.agent("bot", region_sql(), rows=rows) + run.code("calc", value, "adj") + run.card(code_metrics(value)(None, None, rows, rows)) + return run + return by_id(run_compare(make("r1", "model-a", BASE_ROWS, 10), make("r2", "model-b", NEXT_ROWS, 12)))["calc"] + + cases = [{"key": "peak", "condition": "true"}] + for extra in (node("route", "branch", "分流", mode="llm", instruction="判断是否旺季", cases=cases), + node("route", "validate", "校验", schema={"type": "object"}), + node("route", "retrieve", "检索", query="旺季", rerank="model")): + found = calc(extra) + assert found["classification"] == "incomparable", (extra["type"], found["reasons"]) + assert found["reasons"] == [f"输入「vars.adj」来自「调整」，上游节点「{extra['data']['label']}」跟随的工作流默认" + "设置（模型）有变化，无法判断变化来源"] + for extra in (node("route", "branch", "分流", mode="expression", cases=cases), + node("route", "validate", "校验", schema={"type": "object"}, repair_with_llm=False), + node("route", "retrieve", "检索", query="旺季")): + found = calc(extra) + assert found["classification"] == "data_only", (extra["type"], found["reasons"]) + + +def test_every_setting_a_node_reads_is_known_to_the_comparison(): + """节点运行时按 NodeContext.cfg 读配置，节点上没写就回退到图级默认设置。对比按「哪类节点读哪几项」把默认设置的 + 变化落到节点上（compare._CFG_KEYS）：执行器新读了一项、这里没登记，默认设置改了这一项就会被漏掉。""" + by_module: dict = {} + for kind, runner in RUNNERS.items(): + by_module.setdefault(inspect.getmodule(runner), set()).add(str(kind)) + for module, kinds in by_module.items(): + source = inspect.getsource(module) + read = set(re.findall(r'ctx\.cfg\(\s*"(\w+)"', source)) + read |= set(re.findall(r'_setting\(spec, node, "(\w+)"', source)) + if "ctx.approval_mode()" in source: + read.add("approval") # NodeContext.approval_mode 同样先读节点、再回退到默认设置 + known = set().union( + *(compare._CFG_KEYS.get(k, frozenset()) | compare._CFG_KEYS_IF.get(k, frozenset()) for k in kinds)) + assert read <= known, (module.__name__, sorted(read - known)) + + def cited(run_id, rows=BASE_ROWS, *, column="gmv", span=None): """Agent 按出处交回数组字段：引用快照的哪一列、哪一段行由调用方定。""" run = Fake(run_id) @@ -725,6 +896,78 @@ def test_an_agent_citing_other_rows_is_not_a_data_change(): assert total["classification"] == "data_only" and total["attribution"]["rows"] +#: 按区域排序的结果这次多了北区，排在南区前面，西区挪到第 4 行 +WITH_NORTH = [["东区", 1200.5, 100.0], ["北区", 90.0, 0.0], ["南区", 800.0, 50.0], ["西区", 600.0, 0.0]] + + +def test_the_same_rows_are_not_the_same_citation_when_only_one_side_is_the_whole_result(): + """Agent 两次都引用第 1 至 3 行：基准运行里那是整份结果，这次西区还在第 4 行、没被引用。行段一样也不是同一处， + 不能把西区算成「数据没了」（占比也会超过 100%）；反过来（基准只引用了一部分、这次是整份）也一样。""" + total = by_id(run_compare(cited("r1"), cited("r2", WITH_NORTH, span=(0, 2))))["total"] + assert total["classification"] == "incomparable" and total["attribution"] is None + assert total["reasons"] == ["Agent「区域分析」在基准运行中引用了整份结果，这次只引用了第 1 至 3 行（结果共 4 行），" + "取的不是同一组数"] + total = by_id(run_compare(cited("r1", WITH_NORTH, span=(0, 2)), cited("r2")))["total"] + assert total["classification"] == "incomparable" and total["attribution"] is None + assert total["reasons"] == ["Agent「区域分析」在基准运行中只引用了第 1 至 3 行（结果共 4 行），这次引用了整份结果，" + "取的不是同一组数"] + # 两边都只引用了前 3 行、都不是整份结果：同一处，照常是数据变化 + later = [["东区", 1500.5, 120.0], ["北区", 95.0, 0.0], ["南区", 700.0, 40.0], ["西区", 600.0, 10.0]] + total = by_id(run_compare(cited("r1", WITH_NORTH, span=(0, 2)), cited("r2", later, span=(0, 2))))["total"] + assert total["classification"] == "data_only" and total["attribution"]["rows"] + + +def test_cited_rows_on_an_unreadable_snapshot_are_not_taken_as_the_same(): + """基准运行 Agent 那次查询的快照文件不在了：两边都引用第 1 至 3 行，可基准那次是不是整份结果核对不了，不报成 + 数据变化。""" + base = cited("r1") + del base.store[base.queries[1]] + total = by_id(run_compare(base, cited("r2", WITH_NORTH, span=(0, 2))))["total"] + assert total["classification"] == "incomparable" + assert total["reasons"] == ["基准运行的查询快照无法读取，无法核对 Agent「区域分析」引用的是不是整份结果"] + # 这次引用的是整份结果、基准那次的行段不同：同样核对不了是不是两边都是整份 + total = by_id(run_compare(base, cited("r2", WITH_NORTH)))["total"] + assert total["classification"] == "incomparable" and "无法核对" in total["reasons"][0] + + +def test_an_element_of_a_cited_array_follows_its_row(): + """口径卡取 Agent 数组字段里的第一个元素：Agent 交的行段换了起点，第一个元素落到了别的行，不是同一个数；起点没变 + 的照常是数据变化。""" + def make(run_id, rows, span): + run = Fake(run_id) + run.tool("fetch", region_sql(), rows=rows) + bot = run.agent("bot", region_sql(), rows=rows, span=span) + row = span[0] + item = {"path": "vars.kpi.amounts[0]", "value": rows[row][1], "node_id": "bot", "via": "agent_field", + "field": "amounts[0]", "ref": f"Q1.r{row}.gmv", "artifact": bot, + "locator": {"row": row, "column": "gmv"}, "eid": f"ev:cell:{bot[:8]}:{row}:gmv", "status": "verified"} + run.card([metric("first", "vars.kpi.amounts[0]", rows[row][1], [item])]) + return run + + first = by_id(run_compare(make("r1", BASE_ROWS, (0, 2)), make("r2", NEXT_ROWS, (1, 2))))["first"] + assert first["classification"] == "incomparable" + assert first["reasons"] == ["Agent「区域分析」引用的行由第 1 行改为第 2 行，取的不是同一个数"] + first = by_id(run_compare(make("r1", BASE_ROWS, (0, 2)), make("r2", NEXT_ROWS, (0, 2))))["first"] + assert (first["classification"], first["reasons"]) == ("data_only", [compare.DATA_FROM_QUERIES]) + + +def test_no_split_when_an_old_leaf_now_cites_another_column(): + """口径变了（v1 → v2），旧口径用到的 Agent 字段这次引用的是退款列：说不清它只是数据变了，不拆，说明为什么。""" + def make(run_id, column, expression, factor, version): + run = Fake(run_id) + run.tool("fetch", region_sql()) + bot = run.agent("bot", region_sql(), column=column) + value = sum(r[COLUMNS.index(column)] for r in BASE_ROWS) * factor + run.card([metric("total", expression, value, [amounts_in(bot, BASE_ROWS, column=column)])], version=version) + return run + + total = by_id(run_compare(make("r1", "gmv", "sum(vars.kpi.amounts)", 1, "v1"), + make("r2", "refund", "sum(vars.kpi.amounts) * 2", 2, "v2")))["total"] + assert total["classification"] == "caliber_and_data" and total["decomposition"] is None + assert total["decomposition_note"].startswith("无法确认旧口径用到的输入只是数据有变化") + assert "refund" in total["decomposition_note"] + + def test_an_unknown_writer_on_one_side_is_named_plainly(): """一边记不出 vars.adj 是哪个节点写的：说清是哪一边没记录，不写「None」。""" def make(run_id, writer, value): @@ -751,11 +994,32 @@ def test_a_list_run_input_is_compared_by_its_content(): run.card([metric("goal", "sum(input.targets)", float(sum(targets)), [item])]) return run goal = by_id(run_compare(make("r1", [100.0, 200.0]), make("r2", [100.0, 500.0])))["goal"] - assert goal["classification"] == "params_only" and any("targets" in r for r in goal["reasons"]), goal["reasons"] + assert goal["classification"] == "params_only" + assert goal["reasons"] == ["运行输入「input.targets」的内容有变化"] goal = by_id(run_compare(make("r1", [100.0, 200.0]), make("r2", [100.0, 200.0])))["goal"] assert goal["classification"] == "unchanged" +def test_a_list_run_input_without_its_full_value_is_not_taken_as_unchanged(): + """运行输入是列表，入口节点当时的产出不在（还原不出完整值），卡上只记了「[2 项]」：大概一样、指标的值却变了， + 核对不了它变没变，不报成数据变化；项数变了的照常是参数变化；指标的值没变的照常没变。""" + def make(run_id, targets): + run = Fake(run_id, inputs={"targets": targets}) + item = {"path": "input.targets", "value": f"[{len(targets)} 项]", "node_id": "start", "via": "input", + "field": "targets", "status": "ok"} + run.card([metric("goal", "sum(input.targets)", float(sum(targets)), [item])]) + return run + + goal = by_id(run_compare(make("r1", [100.0, 200.0]), make("r2", [100.0, 500.0])))["goal"] + assert goal["classification"] == "incomparable" + assert goal["reasons"] == ["运行输入「input.targets」是列表或对象，当时没有保存完整值，无法核对是否有变化"] + goal = by_id(run_compare(make("r1", [100.0, 200.0]), make("r2", [100.0, 200.0, 50.0])))["goal"] + assert goal["classification"] == "params_only" + assert goal["reasons"] == ["运行输入「input.targets」的内容有变化"] + goal = by_id(run_compare(make("r1", [100.0, 200.0]), make("r2", [100.0, 200.0])))["goal"] + assert (goal["classification"], goal["reasons"]) == ("unchanged", []) + + def test_a_run_input_change_is_a_parameter_change(): def make(run_id, goal): run = Fake(run_id, inputs={"goal": goal}) @@ -874,9 +1138,25 @@ def test_swapped_queries_with_the_same_literals_pair_up_exactly(): result = run_compare(make("r1", ["东区", "南区"], 10, 1.0), make("r2", ["南区", "东区"], 12, 2.0)) assert [(q["structure"], q["literals"]) for q in result["queries"]] == [("same", []), ("same", [])] + # Agent 两次交出的一模一样（只是查询的先后换了）：它没带进模型本身的差异 assert by_id(result)["calc"]["classification"] == "data_only" +def test_renamed_nodes_pair_their_queries_by_literals_first(): + """节点换了 ID，同一个模板查了东区和南区、这次先后互换：跨节点按 SQL 配时同样先配字面量也相同的，两条都没变， + 不会配成两条「参数不同」。""" + def make(run_id, node_id, regions): + run = Fake(run_id) + for region in regions: + run.tool(node_id, f"SELECT SUM(amount) AS gmv FROM orders WHERE region = '{region}'", ["gmv"], [[1.0]], + types={"gmv": "number"}) + return run + + result = run_compare(make("r1", "fetch", ["东区", "南区"]), make("r2", "pull", ["南区", "东区"])) + assert [(q["node_id"], q["structure"], q["literals"]) for q in result["queries"]] == [ + ("pull", "same", []), ("pull", "same", [])] + + def test_a_switched_data_source_is_a_structure_change(): """SQL 一字不差、换了数据源：查的不是同一份数据。""" result = run_compare(week("r1"), week("r2", NEXT_ROWS, source="shop_backup")) @@ -903,6 +1183,17 @@ def test_reformatting_the_workflow_sql_is_not_a_change(): result = run_compare(week("r1"), week("r2", NEXT_ROWS, sql=loose, graph=weekly_graph(loose))) assert {m["classification"] for m in result["metrics"]} == {"data_only"} + # 代码节点在它下游：取数节点不算改了配置，变化照常来自上游数据 + def make(run_id, sql, rows, value): + run = Fake(run_id, graph=weekly_graph(sql, after_fetch=(calc_node(),))) + run.tool("fetch", sql, rows=rows) + run.agent("bot", sql, rows=rows) + run.code("calc", value, "adj") + run.card(code_metrics(value)(None, None, rows, rows)) + return run + calc = by_id(run_compare(make("r1", region_sql(), BASE_ROWS, 10), make("r2", loose, NEXT_ROWS, 12)))["calc"] + assert (calc["classification"], calc["reasons"]) == ("data_only", [compare.DATA_FROM_UPSTREAM_SCOPED]) + def test_renaming_a_node_is_not_a_change(): graph = weekly_graph() @@ -918,9 +1209,10 @@ OUTER_SQL = "SELECT region, gmv FROM m ORDER BY region" MERGED_TYPES = {"region": "text", "gmv": "number"} -def merged(run_id, sql, rows, *, template=None, nested=False, twice=False): +def merged(run_id, sql, rows, *, template=None, nested=False, twice=False, lost=False, bare=False, unsealed=False): """取数节点查一次，合并节点把它当输入 a 再执行一条合并 SQL（nested：再套一层合并；twice：这次多接一个输入 b）， - 口径卡取合并结果的一格。""" + 口径卡取合并结果的一格。追不下去的几种（只用于一层合并）：lost 合并的快照文件不在了；bare 合并没记录输入（快照、 + merge.end 里都没有）；unsealed 上游那次查询不在封存范围内（事件里没有它）。""" nodes = [node("start", "input", "开始"), node("fetch", "tool", "区域取数", tool=TOOL, args={"sql": template or sql}), node("join", "merge", "合并", sql=MERGE_SQL, inputs={"a": "fetch", **({"b": "fetch"} if twice else {})})] if nested: @@ -936,6 +1228,14 @@ def merged(run_id, sql, rows, *, template=None, nested=False, twice=False): types=MERGED_TYPES), "outer" run.card([metric("east", f"cell(nodes.{owner}, 0, 'gmv')", table[0][1], [cell_in(0, "gmv", table[0][1], top, node_id=owner)], name="东区金额")]) + if lost: + del run.store[top] + if bare: + run.store[top] = {k: v for k, v in run.store[top].items() if k != "inputs"} + run.events = [(seq, etype, nid, {k: v for k, v in data.items() if k != "inputs"} + if etype == "merge.end" else data) for seq, etype, nid, data in run.events] + if unsealed: + run.events = [e for e in run.events if e[2] != "fetch"] return run @@ -961,6 +1261,21 @@ def test_a_merged_cell_follows_the_queries_feeding_the_merge(): assert east["east"]["classification"] == "sql_changed" and any("「b」" in r for r in east["east"]["reasons"]) +def test_a_merged_cell_whose_inputs_cannot_be_followed(): + """合并的快照文件不在了：按 merge.end 记的输入接着追，上游加了筛选照样认得出。合并没记录输入、上游那次查询不在 + 封存范围内：说不清上游的取数变没变，不报成数据变化。""" + paid = region_sql(extra=" AND status = 'paid'") + east = by_id(run_compare(merged("r1", region_sql(), BASE_ROWS, lost=True), + merged("r2", paid, NEXT_ROWS, lost=True)))["east"] + assert east["classification"] == "sql_changed", east["reasons"] + assert any("「a」" in r and "区域取数" in r for r in east["reasons"]), east["reasons"] + for broken, said in (({"bare": True}, "没有记录它的输入"), ({"unsealed": True}, "不在封存范围内")): + east = by_id(run_compare(merged("r1", region_sql(), BASE_ROWS, **broken), + merged("r2", region_sql(), NEXT_ROWS, **broken)))["east"] + assert east["classification"] == "incomparable", (broken, east["reasons"]) + assert any(said in r for r in east["reasons"]), (broken, east["reasons"]) + + def test_an_agent_reconfigured_with_the_same_sql_is_a_template_change(): """Agent 改了提示词、SQL 一字不差：它按出处交来的字段可能换了取法，不能报成数据变化。""" graph = weekly_graph() From 36569e1fa458a01453f0a679270ed4fe3de8de1e Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 12:28:38 +0800 Subject: [PATCH 8/8] =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=AF=B9=E6=AF=94?= =?UTF-8?q?=E5=A4=8D=E6=A0=B8=E4=BF=AE=E6=AD=A3=EF=BC=88=E7=AC=AC=203=20?= =?UTF-8?q?=E8=BD=AE=EF=BC=89=EF=BC=9A=E8=A1=8C=E5=8F=B7=E7=9B=B8=E5=90=8C?= =?UTF-8?q?=E7=9A=84=E5=BC=95=E7=94=A8=E8=BF=98=E8=A6=81=E5=BC=95=E7=94=A8?= =?UTF-8?q?=E5=90=8C=E4=B8=80=E6=89=B9=E7=BB=B4=E5=BA=A6=E5=80=BC=EF=BC=8C?= =?UTF-8?q?=E8=AE=B0=E4=B8=8D=E5=87=BA=E5=86=99=E5=85=A5=E8=80=85=E6=97=B6?= =?UTF-8?q?=E5=8F=AA=E7=9C=8B=E5=8F=A3=E5=BE=84=E5=8D=A1=E4=B9=8B=E5=89=8D?= =?UTF-8?q?=E7=9A=84=E8=8A=82=E7=82=B9=EF=BC=8C=E9=BB=98=E8=AE=A4=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E7=9A=84=E7=99=BB=E8=AE=B0=E6=8C=89=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E9=80=90=E7=B1=BB=E6=A0=B8=E5=AF=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Agent 两次引用的行号相同、都不是整份结果时，一边引用了的维度值（区域……）在另一边的快照里却没被引用（按区域排序多了北区、按金额取前 3 名名次变了），判为说不清，说明写出是哪个维度值、在第几行；不再把还在快照里的区域算成「数据没了」、给出远超 100% 的占比。另一边快照里真的没有了的照常是数据变化。数组字段里的单个元素同样核对。维度按两边整份结果认，遮罩列不拿来认，分不出维度的仍按行号认 - 两边都记不出写入者时，范围从整张图改为口径卡之前可能执行过的节点（它的上游加并行支路）：口径卡自己、只在它下游的写周报写不了它读到的值，不再被点名成来源；并行支路上的节点不说成「上游」 - 「整份结果」统一按引用了哪几行判断，容器一支不再单列 - 默认设置登记表的核对测试改为按节点类型：从各执行器出发顺着用到的函数（含就地 import、回调、类）找 cfg 读法，在字面量元组上循环读的键名认得出，认不出的变量键名直接失败；核对节点模块里每个读配置的函数都走到了，登记了不读的也算错。据此补登记成果节点出具契约读的报告节点三项（evidence_from、metrics_from、entities） - 补测试守住上一轮的几条规则：Agent 交的不同而代码节点的数没变是没变，只有这次运行的快照读不出来时说这次，默认设置同时改了结论句裁判时 Agent 只写模型，调不调模型由默认设置决定，报告节点在代码节点上游，改了几个节点时点名画布上靠前的 --- backend/app/engine/compare.py | 179 +++++++++++---- backend/tests/test_compare_engine.py | 311 ++++++++++++++++++++++++--- 2 files changed, 417 insertions(+), 73 deletions(-) diff --git a/backend/app/engine/compare.py b/backend/app/engine/compare.py index b727210a..81b7e701 100644 --- a/backend/app/engine/compare.py +++ b/backend/app/engine/compare.py @@ -23,10 +23,12 @@ 结果时，再按别名一路追到喂它的每一份上游快照。只差字面量时还要看图里的 SQL：图里的 SQL 改过（改了写死的筛选值、 系数）是模板变化，图里一字未改的才是运行时代入的参数 - Agent 按出处交来的字段还要引用同一列、同一段行（两边都引用整份结果的不算换了；行段相同、一边是整份结果一边 - 不是的算换了）：取哪一列、哪几行是 Agent 每次运行时定的，换了就不是同一个数 + 不是的算换了；行号相同、都不是整份结果的，一边引用了的维度值在另一边的快照里、却没被引用的也算换了——取前几名 + 时名次变了、按区域排序时多了一个区域）：取哪一列、哪几行是 Agent 每次运行时定的，换了就不是同一个数 - 不落在快照上的：模型、人交出的值（Agent 没按出处交、模型调用、人工审批……）没有出处，值变了说不清是数据还是它 - 本身的差异；代码、数据整形这类节点看它自己查过的库，不查库的看它的配置和画布上的全部上游。上游的查询结果两次 - 一模一样、值却变了的同样说不清；上游有模型、人交出值的节点、两次交出的不同的也说不清 + 本身的差异；代码、数据整形这类节点看它自己查过的库，不查库的看它的配置和画布上的全部上游；记不出写入者的看 + 口径卡之前可能执行过的节点（口径卡自己、只在它下游的不算）。上游的查询结果两次一模一样、值却变了的同样说不清； + 上游有模型、人交出值的节点、两次交出的不同的也说不清 - 图级的变化落到节点上：连线有增删的，终点节点算改了；默认设置改了的，运行时会读这几项（_CFG_KEYS）、又没有自行 设置的节点算改了 - 严格拆分（只对口径变了的）：旧表达式的每个叶子路径都能在这次的口径卡输入里取得值时， @@ -97,10 +99,12 @@ _MODEL_KEYS = frozenset({"provider", "model", "temperature", "max_tokens", "thin #: 各类节点运行时按 NodeContext.cfg 读的配置项：节点上没写这一项就回退到图级默认设置（GraphSpec.defaults），默认 #: 设置改了这一项，没自行设置它的这类节点交出的值可能跟着变；不读它的不受影响（结论句裁判只有报告节点读）。读了也 #: 不影响交出的值的见 _QUIET_DEFAULTS。合并查询、双盲复算、反方审稿不经 cfg 读配置。执行器新读了一项要在这里登记， -#: tests/test_compare_engine.py 按源码核对 +#: tests/test_compare_engine.py 按各类节点的执行器（连同它用到的函数）的源码逐类核对 _CFG_KEYS: dict[str, frozenset[str]] = { "input": frozenset({"fields"}), - "output": frozenset({"fields", "contract"}), + # 出具契约按报告节点同一套参数重建证据目录：后三项读的是报告节点上的，报告节点没写同样回退到默认设置。成果节点 + # 自己从不写这三项，默认设置改了就算它跟着变（报告节点自己写了的也算，宁可多算） + "output": frozenset({"fields", "contract", "evidence_from", "metrics_from", "entities"}), "transform": frozenset({"mode", "expression", "template", "assign_to"}), "llm": _MODEL_KEYS | {"system", "skills", "use_history", "prompt", "output_schema", "emit_message", "assign_to"}, "agent": _MODEL_KEYS | {"system", "skills", "use_history", "prompt", "output_schema", "cite_fields", "tools", @@ -691,6 +695,20 @@ def _worst(changes: Iterable[_Change]) -> _Change: return max(found, key=lambda c: _WEIGHT[c.kind]) if found else _Change("same") +def _reach(node_id: str, links: Mapping[str, set[str]], memo: dict[str, set[str]]) -> set[str]: + """顺着连线（links：节点 → 上一层或下一层的节点）能走到的全部节点，不含它自己（成环的除外）。""" + if node_id not in memo: + seen: set[str] = set() + todo = [node_id] + while todo: + for nxt in links.get(todo.pop(), ()): + if nxt not in seen: + seen.add(nxt) + todo.append(nxt) + memo[node_id] = seen + return memo[node_id] + + def _from_input(path: str, item: dict[str, Any]) -> bool: head = _REF_HEAD.match(path) return item.get("via") == "input" or (head is not None and head.group(1) == "input") @@ -709,6 +727,7 @@ class _Compare: self._norm: dict[tuple[str, str | None], NormalizedSql] = {} self._shapes: dict[tuple[int, int], _Shape] = {} self._ancestors: dict[str, set[str]] = {} + self._descendants: dict[str, set[str]] = {} self.base_nodes, self.target_nodes = _nodes(base.graph), _nodes(target.graph) self.base_queries, self.target_queries = _queries(base), _queries(target) self.pairs = _pair(self.base_queries, self.target_queries, self._norm_of) @@ -719,8 +738,10 @@ class _Compare: self.order = {n: i for i, n in enumerate(dict.fromkeys([*self.target_nodes, *self.base_nodes]))} self.base_outputs, self.target_outputs = _outputs(base), _outputs(target) self.parents: dict[str, set[str]] = {} + self.children: dict[str, set[str]] = {} for source, target_id, _ in _edges(base.graph) | _edges(target.graph): self.parents.setdefault(target_id, set()).add(source) + self.children.setdefault(source, set()).add(target_id) # ---------------------------------------------------------------- 结果 @@ -921,24 +942,26 @@ class _Compare: def _upstream(self, node_id: str) -> set[str]: """画布上 node_id 的全部上游（两次运行的图合起来，宁可多算）。""" - if node_id not in self._ancestors: - seen: set[str] = set() - todo = [node_id] - while todo: - for parent in self.parents.get(todo.pop(), ()): - if parent not in seen: - seen.add(parent) - todo.append(parent) - self._ancestors[node_id] = seen - return self._ancestors[node_id] + return _reach(node_id, self.parents, self._ancestors) + + def _downstream(self, node_id: str) -> set[str]: + """画布上 node_id 的全部下游（两次运行的图合起来）。""" + return _reach(node_id, self.children, self._descendants) + + def _before(self, card: str) -> set[str]: + """口径卡执行之前可能执行过的节点：画布上它的上游，加上和它不在一条线上的（并行的支路可能先写了变量）。它自己、 + 只在它下游的节点在它之后才执行，写不了它读到的值（循环回边连回来的算上游）。""" + after = self._downstream(card) - self._upstream(card) + return {n for n in self.titles if n != card and n not in after} def _type_of(self, node_id: str) -> str | None: node = self.target_nodes.get(node_id) or self.base_nodes.get(node_id) return str(node.get("type")) if node is not None else None - def _moved(self, node_id: str) -> str: - """一个影响取值的节点变化（self.changed）写成一句话。""" - name = f"上游节点「{self.title(node_id)}」" + def _moved(self, node_id: str, *, upstream: bool = True) -> str: + """一个影响取值的节点变化（self.changed）写成一句话。upstream 是它在不在画布上的上游（并行支路上的节点不在， + 不说成「上游」）。""" + name = f"{'上游节点' if upstream else '节点'}「{self.title(node_id)}」" change = self.changed.get(node_id, "") if change == "defaults": keys = self.inherited.get(node_id, []) @@ -959,9 +982,9 @@ class _Compare: before, after = self.base_outputs.get(node_id, ()), self.target_outputs.get(node_id, ()) return before != after or None in before or None in after - def provenance(self, path: str, b: Any, t: Any, *, moved: bool = True) -> _Change: - """一个输入的出处在两次运行之间变没变（b、t 是推演重算还原出来的输入）。moved 是这个指标的值变没变：输入是 - 容器、完整值又还原不出来，核对不了它变没变时，按它推断。""" + def provenance(self, path: str, b: Any, t: Any, *, card: str, moved: bool = True) -> _Change: + """一个输入的出处在两次运行之间变没变（b、t 是推演重算还原出来的输入，card 是读它的口径卡节点）。moved 是这个 + 指标的值变没变：输入是容器、完整值又还原不出来，核对不了它变没变时，按它推断。""" if b is None or t is None: return _Change("unknown", f"输入「{path}」只在一边的口径卡里有记录") if b.snapshot or t.snapshot: @@ -992,21 +1015,30 @@ class _Compare: return own if writer and writer in self.changed: return _Change("template", self._moved(writer)) - # 写它的节点不查库（代码、数据整形……）：看画布上它的全部上游；写它的节点不知道就看整张图 - scope = self._upstream(writer) if writer else set(self.titles) + # 写它的节点不查库（代码、数据整形……）：看画布上它的全部上游。写它的节点不知道时，看口径卡之前可能执行过的 + # 节点（_before）：口径卡自己、只在它下游的节点（写周报……）写不了它读到的值，点名它们就把下游的变化说成了 + # 它的来源。above 是点名时算「上游」的节点 + if writer: + scope = above = self._upstream(writer) + else: + scope, above = self._before(card), self._upstream(card) moved_nodes = self._ordered(n for n in scope if n in self.changed) worst = _worst(self._query_change(b2, t2) for b2, t2 in self.pairs if (t2 or b2).node_id in scope) where = f"来自「{self.title(writer)}」" if writer else "不知道由哪个节点写入" if moved_nodes: - return _Change("unknown", f"输入「{path}」{where}，{self._moved(moved_nodes[0])}，无法判断变化来源") + first = moved_nodes[0] + return _Change("unknown", f"输入「{path}」{where}，{self._moved(first, upstream=first in above)}，" + "无法判断变化来源") if worst.kind in ("sql", "template", "unknown"): return _Change("unknown", f"输入「{path}」{where}，对不上具体的查询；它的上游有变化，无法判断变化来源") if not same and (spoken := self._ordered(n for n in scope if self._regenerated(n))): # 上游有模型、人交出的值，两次不同：它交的数变了，可能是数据变了，也可能是模型本身的差异（它可能用的 # 正是 Agent 没按出处交的数）。上游的查询结果变没变都一样说不清 - kind = TYPE_LABEL.get(self._type_of(spoken[0]) or "", "节点") - return _Change("unknown", f"输入「{path}」{where}，它的上游节点「{self.title(spoken[0])}」（{kind}）两次" - "交出的结果不同，无法判断变化是否来自数据") + first = spoken[0] + who = ("它的上游节点" if writer else "上游节点") if first in above else "节点" + kind = TYPE_LABEL.get(self._type_of(first) or "", "节点") + return _Change("unknown", f"输入「{path}」{where}，{who}「{self.title(first)}」（{kind}）两次交出的结果不同，" + "无法判断变化是否来自数据") if worst.kind == "same" and not same and not self._data_moved(scope): # 上游查到的数据两次一模一样，它交的值却变了：不是数据变的。上游只有运行输入变了的，算参数 if (self.base.inputs or {}) != (self.target.inputs or {}) \ @@ -1039,7 +1071,8 @@ class _Compare: def _citation(self, agent: Any, b: Any, t: Any, qb: _Query, qt: _Query) -> _Change | None: """Agent 按出处交来的字段两次引用的是不是同一处：同一列（对象数组是同一组列）、同一行或同一段行。两边引用的都是 整份结果的算同一处（这次多了几行也是）；行段相同、一边是整份结果一边不是的不算——没引用的那几行还在这次的快照 - 里，按维度配对会把它们算成「数据没了」。不是同一处、核对不了是不是同一处的，返回说明。""" + 里，按维度配对会把它们算成「数据没了」。行号相同、都不是整份结果的，还要引用同一批维度值（_left_out）。不是 + 同一处、核对不了是不是同一处的，返回说明。""" name = f"Agent「{self.title(agent)}」" if agent else "Agent" lb, lt = _locator(b), _locator(t) cb, ct = lb.get("column") or lb.get("columns"), lt.get("column") or lt.get("columns") @@ -1050,11 +1083,13 @@ class _Compare: if b.row is not None or t.row is not None: # 单格（数组字段里的一个元素、按出处交的一格）：取哪一行由口径卡的下标和 Agent 交的行段一起定，同一行才是 # 同一处 - return None if (b.row, b.rows) == (t.row, t.rows) else changed + return self._left_out(name, what, b, t, qb, qt, cb) if (b.row, b.rows) == (t.row, t.rows) else changed wb, wt = _whole(b, qb), _whole(t, qt) same_span = b.rows == t.rows - if (wb and wt) or (same_span and wb is False and wt is False): + if wb and wt: return None + if same_span and wb is False and wt is False: + return self._left_out(name, what, b, t, qb, qt, cb) if (wb is None or wt is None) and (same_span or (wb is not False and wt is not False)): # 快照读不出来，核对不了是不是整份结果：行段相同的可能一边整份一边不是，行段不同的也可能两边都是整份 side = "基准运行" if wb is None else "这次运行" @@ -1067,6 +1102,45 @@ class _Compare: return _Change("unknown", f"{name}在基准运行中只引用了{_rows_text(b)}（结果共 {len(qb.snap['rows'])} 行），" "这次引用了整份结果，取的不是同一组数") + def _left_out(self, name: str, what: str, b: Any, t: Any, qb: _Query, qt: _Query, column: Any) -> _Change | None: + """行号相同、都不是整份结果的引用，引用的是不是同一批维度值（区域……）。一边引用了的，另一边没引用、却还在那边的 + 快照里，取的就不是同一组数：按区域排序的结果多了北区，西区挪出了引用的行段；按金额取前 3 名，北区挤了进来—— + 按维度配对会把还在的西区算成「数据没了」。另一边的快照里也没有的，是数据真的没了，照常比。维度按两边整份结果 + 认（和归因同一个规则，_dimensions）；分不出维度（没有不是数值的列、维度列设了遮罩）、快照读不出来、行的形状 + 不对的，只能按行号认，返回 None。""" + if qb.snap is None or qt.snap is None: + return None + rb, rt = _records(qb.snap, None), _records(qt.snap, None) + if rb is None or rt is None: + return None + dims, _ = _dimensions(qb.snap, qt.snap, rb, rt, column if isinstance(column, str) else "", self._hidden(b, t)) + if not dims: + return None + + def split(records: list[tuple[int, dict[str, Any]]], inp: Any) -> tuple[dict[Any, int], dict[Any, int]]: + """{维度键: 第一次出现的行号}：引用的行里的，引用的行外面的。""" + first, last = _cited(inp, len(records)) + inside: dict[Any, int] = {} + outside: dict[Any, int] = {} + for i, row in records: + (inside if first <= i <= last else outside).setdefault(tuple(_hashable(row.get(d)) for d in dims), i) + return inside, outside + + (in_b, out_b), (in_t, out_t) = split(rb, b), split(rt, t) + for key in in_b: + if key not in in_t and key in out_t: + return _Change("unknown", f"{name}两次都引用{_rows_text(t)}，{_key_text(key)}这次仍在查询结果第 " + f"{out_t[key] + 1} 行，不在引用的行里，取的不是{what}") + for key in in_t: + if key not in in_b and key in out_b: + return _Change("unknown", f"{name}两次都引用{_rows_text(b)}，{_key_text(key)}在基准运行中位于查询结果第 " + f"{out_b[key] + 1} 行，不在引用的行里，取的不是{what}") + return None + + def _hidden(self, b: Any, t: Any) -> frozenset[str]: + """两边输入落在的快照上要遮的列（小写）。""" + return (self.base.masked.get(b.snapshot) or frozenset()) | (self.target.masked.get(t.snapshot) or frozenset()) + def _generated(self, path: str, writer: str, same: bool) -> _Change: """模型、人交出的值（Agent 没按出处交、模型调用、人工审批……）：没有出处，值变了说不清是数据变了还是它本身的 差异——同样的数据、同样的配置，模型两次交出的数也可能不同。它自己查的库结构变了、它的配置变了照常标出。""" @@ -1241,7 +1315,7 @@ class _Compare: moved = not recompute._same(bm.get("value"), tm.get("value")) changes: dict[str, _Change] = {} if lineage and bp is not None and tp is not None: - changes = {p: self.provenance(p, bp.inputs.get(p), tp.inputs.get(p), moved=moved) + changes = {p: self.provenance(p, bp.inputs.get(p), tp.inputs.get(p), card=card_id, moved=moved) for p in dict.fromkeys([*tp.inputs, *bp.inputs])} if caliber: # 口径优先；喂它的取数也变了、说不清的，接在口径的说明后面照样写出来，不因为口径变了就不提。只在一边才有 @@ -1276,7 +1350,7 @@ class _Compare: if metric.get("status") == "missing_input": out["reasons"].append(f"{side}这个指标缺少输入，没有值") if out["classification"] in _ATTRIBUTED and bp is not None and tp is not None: - out["attribution"] = self._attribution(bp, tp) + out["attribution"] = self._attribution(bp, tp, card_id) if any(inp.masked for plan in (bp, tp) if plan is not None for inp in plan.inputs.values()): self.masked_seen = True return out @@ -1325,7 +1399,7 @@ class _Compare: if any(inp.masked for inp in used) or any(inp.masked for plan in plans for inp in plan.inputs.values()): return None, SPLIT_MASKED for path in leaves: - change = self.provenance(path, bp.inputs.get(path), t.inputs[path]) + change = self.provenance(path, bp.inputs.get(path), t.inputs[path], card=t.node_id) if change.kind in ("sql", "template"): return None, f"旧口径用到的输入取数也有变化，无法严格拆分：{change.reason}" if change.kind == "unknown": @@ -1375,7 +1449,7 @@ class _Compare: # ---------------------------------------------------------------- 维度归因 - def _attribution(self, bp: Any, tp: Any) -> dict[str, Any] | None: + def _attribution(self, bp: Any, tp: Any, card: str) -> dict[str, Any] | None: """挑一个落在查询快照上、两边都变了的输入做维度归因：先找给得出占比的，没有就取第一个列得出行的，再没有 就取第一个说明（截断、度量列遮罩……）。一个都挑不出来返回 None。""" found: list[dict[str, Any]] = [] @@ -1383,7 +1457,7 @@ class _Compare: b = bp.inputs.get(path) if b is None or not (b.snapshot and t.snapshot) or recompute._same(b.value, t.value): continue - if self.provenance(path, b, t).kind not in ("same", "params"): + if self.provenance(path, b, t, card=card).kind not in ("same", "params"): continue result = self._attribute(b, t) if result is None: @@ -1409,12 +1483,12 @@ class _Compare: if snap.get("truncated"): # 截断的结果只是前几行：各键的和不是完整的和，按它归因会把没取回的行的变化算到别的键头上 return {**empty, "note": TRUNCATED.format(side=side, rows=len(snap["rows"]))} - hidden = (self.base.masked.get(b.snapshot) or frozenset()) | (self.target.masked.get(t.snapshot) or frozenset()) + hidden = self._hidden(b, t) if any(str(c).lower() in hidden for c in [*(sb.get("columns") or []), *(st.get("columns") or [])]): self.masked_seen = True if measure.lower() in hidden: return {**empty, "note": MASKED_MEASURE.format(column=measure)} - rb, rt = _records(sb, b), _records(st, t) + rb, rt = _records(sb, b.rows), _records(st, t.rows) if rb is None or rt is None: return None values = (_input_total(b), _input_total(t)) @@ -1546,14 +1620,28 @@ def _same_input(b: Any, t: Any) -> bool | None: return recompute._same(b.item.get("value"), t.item.get("value")) +def _cited(inp: Any, count: int) -> tuple[int, int]: + """输入引用了快照的哪几行（第一行, 最后一行），count 是快照共几行：单格是那一行，数组字段是它那一段行，容器本身 + 是一份查询结果的是全部的行。""" + if inp.row is not None: + return inp.row, inp.row + if inp.rows is not None: + return inp.rows + return 0, count - 1 + + def _whole(inp: Any, q: _Query) -> bool | None: - """这个输入引用的是不是整份查询结果：容器本身就是整份结果，或者数组字段的行段从第一行到最后一行。快照读不出来、 - 不知道结果共几行的返回 None。""" - if inp.table: - return True + """这个输入引用的是不是整份查询结果：引用的行从第一行到最后一行（容器本身就是整份结果）。快照读不出来、不知道 + 结果共几行的返回 None。""" if q.snap is None: return None - return inp.rows is not None and inp.rows == (0, len(q.snap["rows"]) - 1) + count = len(q.snap["rows"]) + return _cited(inp, count) == (0, count - 1) + + +def _key_text(key: tuple[Any, ...]) -> str: + """维度值写进说明：几列的用「 / 」连起来，和对比页上归因行的写法一样。""" + return "「" + " / ".join(_shown(v) for v in key) + "」" def _rows_text(inp: Any) -> str: @@ -1578,11 +1666,12 @@ def _merge_inputs(q: _Query) -> dict[str, str] | None: if isinstance(i, dict) and i.get("alias") and i.get("artifact")} -def _records(snap: dict[str, Any], inp: Any) -> list[tuple[int, dict[str, Any]]] | None: - """归因用到的行：(行号, {列: 值})。数组字段只用它那一段行，单格用整份结果。""" +def _records(snap: dict[str, Any], span: tuple[int, int] | None) -> list[tuple[int, dict[str, Any]]] | None: + """快照里的行：(行号, {列: 值})。span 给了只取那一段行（归因时数组字段只用它那一段），None 是整份结果（单格按 + 整份结果归因）。行的形状不对返回 None。""" columns = [str(c) for c in snap.get("columns") or []] rows = snap.get("rows") or [] - first, last = (inp.rows if inp.rows is not None else (0, len(rows) - 1)) + first, last = span if span is not None else (0, len(rows) - 1) out = [] for i in range(first, min(last, len(rows) - 1) + 1): row = rows[i] diff --git a/backend/tests/test_compare_engine.py b/backend/tests/test_compare_engine.py index 374ad4ae..0417a722 100644 --- a/backend/tests/test_compare_engine.py +++ b/backend/tests/test_compare_engine.py @@ -10,20 +10,27 @@ SQL，Agent 节点 bot 自己再查一次、把各区域的金额作为数组字 - 遮罩列的值绝不出现在结果里；用到遮罩列的指标不拆分 - 模板对比：新增、删除的节点，配置有变化的节点和配置项名 - 输入的出处：合并查询追到它的上游查询；Agent 按出处交的字段还要引用同一列、同一段行（行段相同、一边是整份结果 - 一边不是的不算）；没按出处交的数、上游数据没变数却变了的、上游的 Agent 两次交出的结果不同的，说不清变化从哪来 - （incomparable）；图里写死的字面量改了、连线变了、读到的默认设置变了都算模板变化 + 一边不是的不算；都不是整份结果的，一边引用了的区域另一边还在快照里却没引用的也不算）；没按出处交的数、上游数据 + 没变数却变了的、上游的 Agent 两次交出的结果不同的，说不清变化从哪来（incomparable），记不出写入者的只看口径卡 + 之前可能执行过的节点；图里写死的字面量改了、连线变了、读到的默认设置变了都算模板变化 """ from __future__ import annotations +import ast import copy +import importlib import inspect import json -import re +import pkgutil +import textwrap from decimal import Decimal +from types import SimpleNamespace +import app.engine.nodes as nodes_package from app.engine import compare from app.engine.compiler import RUNNERS from app.engine.merge_query import MERGE_SOURCE +from app.engine.schema import NodeType from tests.fixtures.compare_runs import ( BASE_ROWS, COLUMNS, @@ -610,6 +617,9 @@ def test_a_code_node_after_an_agent_that_answered_differently_is_incomparable(): assert calc["classification"] == "incomparable" and calc["attribution"] is None assert calc["reasons"] == ["输入「vars.adj」来自「调整」，它的上游节点「区域分析」（Agent）两次交出的结果不同，" "无法判断变化是否来自数据"] + # Agent 交的不同，代码节点交的数两次一样：照常没变，没有可说不清的变化 + calc = by_id(run_compare(make("r1", BASE_ROWS, 0.46, value=1.0), make("r2", NEXT_ROWS, 0.52, value=1.0)))["calc"] + assert (calc["classification"], calc["reasons"]) == ("unchanged", []) # Agent 的产出两次都没落成工件（写工件失败）：一样不一样核对不了，同样不报成数据变化 def unstored(run): @@ -622,20 +632,89 @@ def test_a_code_node_after_an_agent_that_answered_differently_is_incomparable(): def test_a_reconfigured_node_upstream_of_a_code_node_is_named(): - """代码节点上游的数据整形节点改了表达式（SQL 一字没改）：代码节点交的数变了说不清从哪来，说明里点出改的是哪个节点。""" - def make(run_id, expression, rows, value): - shape = node("shape", "transform", "整理", mode="expression", expression=expression, assign_to="kept") - run = Fake(run_id, graph=weekly_graph(after_fetch=(shape, calc_node()))) + """代码节点上游的数据整形节点改了表达式（SQL 一字没改）：代码节点交的数变了说不清从哪来，说明里点出改的是哪个节点； + 改了几个的，点名画布上最靠前的那个（节点 ID 的先后和画布上的先后不同）。""" + def make(run_id, expressions, rows, value): + shapes = [node(nid, "transform", label, mode="expression", expression=expression, assign_to=f"kept_{nid}") + for (nid, label), expression in zip((("shape_m", "整理"), ("shape_z", "再整理"), ("shape_a", "汇总")), + expressions)] + run = Fake(run_id, graph=weekly_graph(after_fetch=(*shapes, calc_node()))) run.tool("fetch", region_sql(), rows=rows) run.agent("bot", region_sql(), rows=rows) run.code("calc", value, "adj") run.card(code_metrics(value)(None, None, rows, rows)) return run - calc = by_id(run_compare(make("r1", "nodes.fetch.rows", BASE_ROWS, 10), - make("r2", "nodes.fetch.rows[:2]", NEXT_ROWS, 12)))["calc"] + same = ["nodes.fetch.rows"] * 3 + calc = by_id(run_compare(make("r1", same, BASE_ROWS, 10), + make("r2", ["nodes.fetch.rows[:2]", *same[1:]], NEXT_ROWS, 12)))["calc"] assert calc["classification"] == "incomparable" assert calc["reasons"] == ["输入「vars.adj」来自「调整」，上游节点「整理」的配置有变化，无法判断变化来源"] + calc = by_id(run_compare(make("r1", same, BASE_ROWS, 10), + make("r2", ["nodes.fetch.rows[:2]"] * 3, NEXT_ROWS, 12)))["calc"] + assert calc["reasons"] == ["输入「vars.adj」来自「调整」，上游节点「整理」的配置有变化，无法判断变化来源"] + + +def test_a_report_upstream_of_a_code_node_that_wrote_differently_is_incomparable(): + """代码节点上游有报告撰写节点（文字是模型写的），两次写得不同：代码节点交的数变了，可能来自数据，也可能来自模型 + 本身的差异；两次写得一字不差的（产出工件一样）照常是数据变化。""" + def make(run_id, rows, text, value): + run = Fake(run_id, graph=weekly_graph(after_fetch=(node("draft", "report", "初稿"), calc_node()))) + run.tool("fetch", region_sql(), rows=rows) + run.add("node.started", "draft", {"node_type": "report"}) + run.add("node.finished", "draft", {"artifact": run.put({"text": text})}) + run.agent("bot", region_sql(), rows=rows) + run.code("calc", value, "adj") + run.card(code_metrics(value)(None, None, rows, rows)) + return run + + calc = by_id(run_compare(make("r1", BASE_ROWS, "上周初稿", 10), make("r2", NEXT_ROWS, "本周初稿", 12)))["calc"] + assert calc["classification"] == "incomparable" + assert calc["reasons"] == ["输入「vars.adj」来自「调整」，它的上游节点「初稿」（报告撰写）两次交出的结果不同，" + "无法判断变化是否来自数据"] + calc = by_id(run_compare(make("r1", BASE_ROWS, "初稿", 10), make("r2", NEXT_ROWS, "初稿", 12)))["calc"] + assert (calc["classification"], calc["reasons"]) == ("data_only", [compare.DATA_FROM_UPSTREAM]) + + +def test_an_unknown_writer_is_looked_for_only_among_nodes_before_the_card(): + """两边都记不出 vars.adj 是哪个节点写的：只看口径卡之前可能执行过的节点。口径卡自己改了配置、它下游的写周报两次写得 + 不同，都写不了它读到的值，不能点名成它的上游；它上游的节点改了配置照常说不清，并行支路上的节点也可能先写了变量， + 同样说不清，只是不说成「上游」。""" + def make(run_id, rows, value, *, other="1", report="上周", reshape="nodes.fetch.rows", aside="nodes.fetch.rows"): + graph = chain(node("start", "input", "开始"), + node("fetch", "tool", "区域取数", tool=TOOL, args={"sql": region_sql()}), + node("shape", "transform", "整理", mode="expression", expression=reshape, assign_to="kept"), + node("calc", "code", "调整", code="x", assign_to="adj"), + node("card", "metrics", "周报口径卡", caliber="周报口径", caliber_version="v1", + metrics=[{"id": "other", "expression": other}]), + node("write", "report", "写周报"), node("out", "output", "成果")) + # 并行支路：开始 → 旁路整理 → 成果，和口径卡不在一条线上 + graph["nodes"].append(node("aside", "transform", "旁路整理", mode="expression", expression=aside, + assign_to="extra")) + graph["edges"] += [{"source": "start", "target": "aside"}, {"source": "aside", "target": "out"}] + run = Fake(run_id, graph=graph) + run.tool("fetch", region_sql(), rows=rows) + run.code("calc", value, "adj") + run.card([metric("calc", "vars.adj * 2", value * 2, [scalar_in("vars.adj", value, None, "state")])]) + run.add("node.started", "write", {"node_type": "report"}) + run.add("node.finished", "write", {"artifact": run.put({"text": report})}) + return run + + # 口径卡自己改了另一个指标的定义，写周报两次写得不同：都在它之后执行，变化照常来自上游数据 + calc = by_id(run_compare(make("r1", BASE_ROWS, 10), make("r2", NEXT_ROWS, 12, other="2", report="本周")))["calc"] + assert (calc["classification"], calc["reasons"]) == ("data_only", [compare.DATA_FROM_UPSTREAM_SCOPED]) + calc = by_id(run_compare(make("r1", BASE_ROWS, 10), make("r2", NEXT_ROWS, 12, report="本周")))["calc"] + assert (calc["classification"], calc["reasons"]) == ("data_only", [compare.DATA_FROM_UPSTREAM]) + # 口径卡上游的整理节点改了表达式 + calc = by_id(run_compare(make("r1", BASE_ROWS, 10), + make("r2", NEXT_ROWS, 12, reshape="nodes.fetch.rows[:2]")))["calc"] + assert calc["classification"] == "incomparable" + assert calc["reasons"] == ["输入「vars.adj」不知道由哪个节点写入，上游节点「整理」的配置有变化，无法判断变化来源"] + # 并行支路上的旁路整理改了表达式 + calc = by_id(run_compare(make("r1", BASE_ROWS, 10), + make("r2", NEXT_ROWS, 12, aside="nodes.fetch.rows[:2]")))["calc"] + assert calc["classification"] == "incomparable" + assert calc["reasons"] == ["输入「vars.adj」不知道由哪个节点写入，节点「旁路整理」的配置有变化，无法判断变化来源"] def test_a_run_input_upstream_of_a_code_node_is_a_parameter_change(): @@ -798,6 +877,11 @@ def test_a_changed_default_model_is_a_template_change_for_agents_that_follow_it( many["defaults"] = {"provider": "openai", "model": "model-b", "temperature": 0.5, "max_tokens": 2000} share = by_id(run_compare(uncited("r1", 0.46, graph=base_graph), uncited("r2", 0.52, graph=many)))["share"] assert share["reasons"] == ["上游节点「区域分析」跟随的工作流默认设置（模型接入、模型、温度等 4 项）有变化"] + # 同时改了 Agent 不读的结论句裁判：只写它跟着变的模型 + judged = copy.deepcopy(target_graph) + judged["defaults"]["judge"] = {"max_cost_usd": 1.0} + share = by_id(run_compare(uncited("r1", 0.46, graph=base_graph), uncited("r2", 0.52, graph=judged)))["share"] + assert share["reasons"] == ["上游节点「区域分析」跟随的工作流默认设置（模型）有变化"] # Agent 自己指定了模型：默认设置变了它不跟着变 for graph in (base_graph, target_graph): graph["nodes"][2]["data"]["config"]["model"] = "model-x" @@ -823,17 +907,19 @@ def test_defaults_that_agents_do_not_read_leave_them_alone(): def test_a_model_default_reaches_every_node_that_calls_the_model(): """默认模型换了，跟着换的不只 Agent：让模型分类的条件分支、让模型修复的结构校验（默认开着）、用模型重排的知识检索 也是。它们在代码节点上游时，代码节点交的数变了不能说来自上游数据；不调模型的照常。""" - def calc(extra): - def make(run_id, model, rows, value): + def calc(extra, before=None, after=None): + def make(run_id, defaults, rows, value): graph = weekly_graph(after_fetch=(extra, calc_node())) - graph["defaults"] = {"provider": "anthropic", "model": model} + graph["defaults"] = defaults run = Fake(run_id, graph=graph) run.tool("fetch", region_sql(), rows=rows) run.agent("bot", region_sql(), rows=rows) run.code("calc", value, "adj") run.card(code_metrics(value)(None, None, rows, rows)) return run - return by_id(run_compare(make("r1", "model-a", BASE_ROWS, 10), make("r2", "model-b", NEXT_ROWS, 12)))["calc"] + before = before or {"provider": "anthropic", "model": "model-a"} + after = after or {"provider": "anthropic", "model": "model-b"} + return by_id(run_compare(make("r1", before, BASE_ROWS, 10), make("r2", after, NEXT_ROWS, 12)))["calc"] cases = [{"key": "peak", "condition": "true"}] for extra in (node("route", "branch", "分流", mode="llm", instruction="判断是否旺季", cases=cases), @@ -849,29 +935,121 @@ def test_a_model_default_reaches_every_node_that_calls_the_model(): found = calc(extra) assert found["classification"] == "data_only", (extra["type"], found["reasons"]) + # 调不调模型也可能由默认设置定：条件分支没写判断方式、默认设置写了让模型判断的，跟着默认模型变；结构校验的默认 + # 设置关了让模型修复的，不跟着变 + asks = node("route", "branch", "分流", instruction="判断是否旺季", cases=cases) + found = calc(asks, {"mode": "llm", "model": "model-a"}, {"mode": "llm", "model": "model-b"}) + assert found["classification"] == "incomparable", found["reasons"] + assert found["reasons"] == ["输入「vars.adj」来自「调整」，上游节点「分流」跟随的工作流默认设置（模型）有变化，" + "无法判断变化来源"] + found = calc(node("route", "validate", "校验", schema={"type": "object"}), + {"repair_with_llm": False, "model": "model-a"}, {"repair_with_llm": False, "model": "model-b"}) + assert found["classification"] == "data_only", found["reasons"] + # 默认的判断方式这次改成了按表达式：基准运行那次让模型判断过，跟着变的是判断方式和模型两项 + found = calc(asks, {"mode": "llm", "model": "model-a"}, {"mode": "expression", "model": "model-b"}) + assert found["reasons"] == ["输入「vars.adj」来自「调整」，上游节点「分流」跟随的工作流默认设置（判断方式、模型）" + "有变化，无法判断变化来源"] + + +#: 只顺着节点模块里的函数往下找：别处的函数不经 NodeContext.cfg 读节点配置 +_NODE_MODULES = "app.engine.nodes." + + +def _defined_here(obj): + """obj 是定义在节点模块里的函数或类（装饰过的取原函数），否则 None。""" + obj = inspect.unwrap(obj) if inspect.isfunction(obj) else getattr(obj, "func", obj) + if (inspect.isfunction(obj) or inspect.isclass(obj)) and obj.__module__.startswith(_NODE_MODULES): + return obj + return None + + +def _settings_read(runner): + """一类节点的执行器运行时按 NodeContext.cfg 读的配置项（节点上没写就回退到默认设置）：执行器自己的函数体，加上它 + 用到的、定义在节点模块里的函数和类，一层层往下找（函数里就地 import 的也算）。返回 (配置项, 认不出键名的读法, + 走到的函数)。键名是变量的，只认在字面量元组上循环出来的（Agent 的预算三项）；别的变量键名认不出是哪几项， + 原样交回，测试据此失败。""" + read, unknown, seen = set(), [], set() + todo = [runner] + while todo: + fn = _defined_here(todo.pop()) + if fn is None or (fn.__module__, fn.__qualname__) in seen: + continue + seen.add((fn.__module__, fn.__qualname__)) + names = vars(importlib.import_module(fn.__module__)) # 它所在模块里的名字（类没有 __globals__） + parts = list(ast.walk(ast.parse(textwrap.dedent(inspect.getsource(fn))))) + imported = {alias.asname or alias.name: (part.module, alias.name) for part in parts + if isinstance(part, ast.ImportFrom) and part.module for alias in part.names} + looped = {} # 循环变量 → 它在字面量元组上取到的键名：{k: ctx.cfg(k) for k in ("max_steps", …)} + for part in parts: + for gen in getattr(part, "generators", None) or ([part] if isinstance(part, ast.For) else []): + if isinstance(gen.target, ast.Name) and isinstance(gen.iter, (ast.Tuple, ast.List)) and all( + isinstance(e, ast.Constant) and isinstance(e.value, str) for e in gen.iter.elts): + looped[gen.target.id] = {e.value for e in gen.iter.elts} + for part in parts: + called = part.func if isinstance(part, ast.Call) else None + if isinstance(called, ast.Attribute) and called.attr == "cfg": + key = part.args[0] if part.args else None + if isinstance(key, ast.Constant) and isinstance(key.value, str): + read.add(key.value) + elif isinstance(key, ast.Name) and key.id in looped: + read |= looped[key.id] + else: + unknown.append(f"{fn.__qualname__}: {ast.unparse(part)}") + elif isinstance(called, ast.Attribute) and called.attr == "approval_mode": + read.add("approval") # NodeContext.approval_mode 同样先读节点、再回退到默认设置 + elif isinstance(called, ast.Name) and called.id == "_setting": + # 报告模块的 _setting(spec, 节点, 键) 和 NodeContext.cfg 同一个规则 + key = part.args[2] if len(part.args) > 2 else None + if isinstance(key, ast.Constant) and isinstance(key.value, str): + read.add(key.value) + else: + unknown.append(f"{fn.__qualname__}: {ast.unparse(part)}") + elif isinstance(part, ast.Name) and isinstance(part.ctx, ast.Load): + # 调用、当回调传出去的函数都算用到了 + target = names.get(part.id) + if target is None and part.id in imported: + module, name = imported[part.id] + target = getattr(importlib.import_module(module), name, None) + todo.append(target) + elif isinstance(part, ast.Attribute) and isinstance(part.value, ast.Name): + module = names.get(part.value.id) + if inspect.ismodule(module): + todo.append(getattr(module, part.attr, None)) + return read, unknown, seen + def test_every_setting_a_node_reads_is_known_to_the_comparison(): """节点运行时按 NodeContext.cfg 读配置，节点上没写就回退到图级默认设置。对比按「哪类节点读哪几项」把默认设置的 - 变化落到节点上（compare._CFG_KEYS）：执行器新读了一项、这里没登记，默认设置改了这一项就会被漏掉。""" - by_module: dict = {} + 变化落到节点上（compare._CFG_KEYS）：执行器新读了一项、这里没登记，默认设置改了这一项就会被漏掉；登记了不读的， + 默认设置改了它就会被误标。按节点类型比，不按模块比：一个模块管几类节点（Agent 和模型调用、调用工具和代码……）， + 其中一类读了、另一类没读的，不能拿另一类的登记充数。""" + reached = set() for kind, runner in RUNNERS.items(): - by_module.setdefault(inspect.getmodule(runner), set()).add(str(kind)) - for module, kinds in by_module.items(): - source = inspect.getsource(module) - read = set(re.findall(r'ctx\.cfg\(\s*"(\w+)"', source)) - read |= set(re.findall(r'_setting\(spec, node, "(\w+)"', source)) - if "ctx.approval_mode()" in source: - read.add("approval") # NodeContext.approval_mode 同样先读节点、再回退到默认设置 - known = set().union( - *(compare._CFG_KEYS.get(k, frozenset()) | compare._CFG_KEYS_IF.get(k, frozenset()) for k in kinds)) - assert read <= known, (module.__name__, sorted(read - known)) + read, unknown, seen = _settings_read(runner) + assert not unknown, (str(kind), unknown) + known = compare._CFG_KEYS.get(str(kind), frozenset()) | compare._CFG_KEYS_IF.get(str(kind), frozenset()) + assert read == known, (str(kind), "没登记", sorted(read - known), "登记了不读", sorted(known - read)) + reached |= seen + # 键名不是字面量的读法也认得出来 + assert {"max_steps", "budget_tokens", "budget_usd"} <= _settings_read(RUNNERS[NodeType.AGENT])[0] + # 节点模块里每个按 cfg 读配置的函数都走到了：走不到的说明上面漏了某种用法，查出来的配置项就不全 + holders = set() + for info in pkgutil.iter_modules(nodes_package.__path__): + module = importlib.import_module(_NODE_MODULES + info.name) + for top in ast.parse(inspect.getsource(module)).body: + if isinstance(top, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and any( + isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) + and n.func.attr in ("cfg", "approval_mode") for n in ast.walk(top)): + holders.add((module.__name__, top.name)) + assert holders and holders <= reached, sorted(holders - reached) -def cited(run_id, rows=BASE_ROWS, *, column="gmv", span=None): +def cited(run_id, rows=BASE_ROWS, *, column="gmv", span=None, sql=None): """Agent 按出处交回数组字段：引用快照的哪一列、哪一段行由调用方定。""" run = Fake(run_id) - run.tool("fetch", region_sql(), rows=rows) - bot = run.agent("bot", region_sql(), rows=rows, column=column, span=span) + sql = sql or region_sql() + run.tool("fetch", sql, rows=rows) + bot = run.agent("bot", sql, rows=rows, column=column, span=span) first, last = span or (0, len(rows) - 1) values = [r[COLUMNS.index(column)] for r in rows[first:last + 1]] run.card([metric("total", "sum(vars.kpi.amounts)", sum(values), @@ -917,6 +1095,65 @@ def test_the_same_rows_are_not_the_same_citation_when_only_one_side_is_the_whole assert total["classification"] == "data_only" and total["attribution"]["rows"] +def test_the_same_rows_are_not_the_same_citation_when_a_cited_region_moved_out_of_them(): + """两边都只引用第 1 至 3 行、都不是整份结果：基准运行引用了西区，这次北区排进了前 3 行，西区挪到第 4 行，还在这次 + 的快照里，只是没被引用。取的不是同一组数，不能把西区算成「数据没了」（占比会远超 100%）；反过来（这次引用的区域 + 在基准运行的快照里、没被引用）也一样。西区这次真的没有了的，照常是数据变化。""" + with_middle = [["东区", 1200.5, 100.0], ["南区", 800.0, 50.0], ["西区", 600.0, 0.0], ["中区", 30.0, 0.0]] + total = by_id(run_compare(cited("r1", with_middle, span=(0, 2)), cited("r2", WITH_NORTH, span=(0, 2))))["total"] + assert total["classification"] == "incomparable" and total["attribution"] is None + assert total["reasons"] == ["Agent「区域分析」两次都引用第 1 至 3 行，「西区」这次仍在查询结果第 4 行，不在引用的" + "行里，取的不是同一组数"] + total = by_id(run_compare(cited("r1", WITH_NORTH, span=(0, 2)), cited("r2", with_middle, span=(0, 2))))["total"] + assert total["classification"] == "incomparable" and total["attribution"] is None + assert total["reasons"] == ["Agent「区域分析」两次都引用第 1 至 3 行，「西区」在基准运行中位于查询结果第 4 行，不在" + "引用的行里，取的不是同一组数"] + + # 按金额从大到小取前 3 名：北区这次挤进了前 3，西区落到第 4 行 + top = region_sql().replace("ORDER BY region", "ORDER BY gmv DESC") + before = [["东区", 1200.0, 0.0], ["南区", 800.0, 0.0], ["西区", 600.0, 0.0], ["中区", 30.0, 0.0]] + after = [["东区", 1200.0, 0.0], ["南区", 800.0, 0.0], ["北区", 700.0, 0.0], ["西区", 600.0, 0.0]] + total = by_id(run_compare(cited("r1", before, span=(0, 2), sql=top), + cited("r2", after, span=(0, 2), sql=top)))["total"] + assert total["classification"] == "incomparable" and total["attribution"] is None + assert "「西区」这次仍在查询结果第 4 行" in total["reasons"][0] + # 对照：西区这次真的没有了（不在这次的快照里），照常是数据变化，按区域归因，占比之和为 1 + gone = [["东区", 1200.0, 0.0], ["南区", 800.0, 0.0], ["北区", 700.0, 0.0], ["中区", 30.0, 0.0]] + total = by_id(run_compare(cited("r1", before, span=(0, 2), sql=top), + cited("r2", gone, span=(0, 2), sql=top)))["total"] + assert (total["classification"], total["reasons"]) == ("data_only", [compare.DATA_FROM_QUERIES]) + rows = total["attribution"]["rows"] + assert [(r["key"], r["base"], r["target"], r["share"]) for r in rows] == [ + (["北区"], None, 700.0, 7.0), (["西区"], 600.0, None, -6.0), (["东区"], 1200.0, 1200.0, 0.0), + (["南区"], 800.0, 800.0, 0.0)] + assert sum(exact(r["share"]) for r in rows) == 1 + + # 区域列设了遮罩：不拿它认是哪一区，只能按行号认，遮住的区域名不出现在结果里 + def masked(run): + run.masked = {a: frozenset({"region"}) for a in run.queries} + return run + result = run_compare(masked(cited("r1", with_middle, span=(0, 2))), masked(cited("r2", WITH_NORTH, span=(0, 2)))) + assert by_id(result)["total"]["classification"] == "data_only" + text = json.dumps(result, ensure_ascii=False) + assert all(region not in text for region in ("东区", "南区", "西区", "北区", "中区")) + + +def test_an_input_cites_the_whole_result_only_when_it_covers_every_row(): + """怎么认整份结果（compare._whole）：容器本身就是一份查询结果的（没有行号、行段）、数组字段的行段从第一行到最后 + 一行的是；单格、只有一段行的不是；快照读不出来的不知道。Agent 的字段在真实运行里总带着行号或行段，容器这一支 + 只是兜底，造不出真实的运行来走它，在这里直接核对。""" + query = compare._Query(artifact="q1", node_id="bot", tool=TOOL, ordinal=1, snap={"rows": [["东区"], ["南区"], ["西区"]]}) + lost = compare._Query(artifact="q1", node_id="bot", tool=TOOL, ordinal=1) + + def cites(row=None, rows=None): + return SimpleNamespace(row=row, rows=rows) + assert compare._whole(cites(), query) is True + assert compare._whole(cites(rows=(0, 2)), query) is True + assert compare._whole(cites(rows=(0, 1)), query) is False + assert compare._whole(cites(row=0), query) is False + assert compare._whole(cites(rows=(0, 2)), lost) is None + + def test_cited_rows_on_an_unreadable_snapshot_are_not_taken_as_the_same(): """基准运行 Agent 那次查询的快照文件不在了：两边都引用第 1 至 3 行，可基准那次是不是整份结果核对不了，不报成 数据变化。""" @@ -929,6 +1166,18 @@ def test_cited_rows_on_an_unreadable_snapshot_are_not_taken_as_the_same(): total = by_id(run_compare(base, cited("r2", WITH_NORTH)))["total"] assert total["classification"] == "incomparable" and "无法核对" in total["reasons"][0] + # 反过来只有这次运行的快照读不出来：说的是这次运行 + def unreadable(run): + del run.store[run.queries[1]] + return run + total = by_id(run_compare(cited("r1"), unreadable(cited("r2", WITH_NORTH, span=(0, 2)))))["total"] + assert total["classification"] == "incomparable" + assert total["reasons"] == ["这次运行的查询快照无法读取，无法核对 Agent「区域分析」引用的是不是整份结果"] + # 基准运行只引用了一部分、这次的行段又不同：这次是不是整份结果都不是同一处，照实说行段换了 + total = by_id(run_compare(cited("r1", WITH_NORTH, span=(0, 2)), unreadable(cited("r2", span=(1, 2)))))["total"] + assert total["classification"] == "incomparable" + assert total["reasons"] == ["Agent「区域分析」引用的行由第 1 至 3 行改为第 2 至 3 行，取的不是同一组数"] + def test_an_element_of_a_cited_array_follows_its_row(): """口径卡取 Agent 数组字段里的第一个元素：Agent 交的行段换了起点，第一个元素落到了别的行，不是同一个数；起点没变 @@ -949,6 +1198,12 @@ def test_an_element_of_a_cited_array_follows_its_row(): assert first["reasons"] == ["Agent「区域分析」引用的行由第 1 行改为第 2 行，取的不是同一个数"] first = by_id(run_compare(make("r1", BASE_ROWS, (0, 2)), make("r2", NEXT_ROWS, (0, 2))))["first"] assert (first["classification"], first["reasons"]) == ("data_only", [compare.DATA_FROM_QUERIES]) + # 起点没变，可这次北区排在了第一行：基准运行取的是东区，东区这次还在第 2 行，取的不是同一个数 + first = by_id(run_compare(make("r1", BASE_ROWS, (0, 2)), + make("r2", [["北区", 90.0, 0.0], *NEXT_ROWS], (0, 3))))["first"] + assert first["classification"] == "incomparable" + assert first["reasons"] == ["Agent「区域分析」两次都引用第 1 行，「东区」这次仍在查询结果第 2 行，不在引用的行里，" + "取的不是同一个数"] def test_no_split_when_an_old_leaf_now_cites_another_column():