mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:31:58 +00:00
运行对比后端（变化三分法）：两次运行按指标拆清是数据变了、口径变了还是模板 / SQL 变了，数据变化按维度归因
GET /api/runs/{id}/compare/base 找默认基准：定时任务发起的先找同一定时任务更早结束的一次，否则同一工作流、
同一运行类别；只挑封存过的终态运行，找不到 404。GET /api/runs/{id}/compare?base= 对比两次运行：两次都要
已结束、封存且核对一致（照搬证据接口的封存判定），否则 409；不是同一个工作流、同一次运行 422。只读，不追加
事件、不落工件。
engine/compare.py 是纯函数：
- 查询按「节点 + 第几次 + 工具」配对，配不上再按规范化 SQL 配；normalize_sql 判 same / params_only /
changed，结构变了的用 diff_facets 写出差在哪（时间范围排在最后）；核查类查询不参与
- 口径卡同一节点按指标 id 配对，表达式、口径版本、小数位、单位、显示格式不同算口径变了
- 每个指标按 added/removed → 口径 → SQL 或上游配置 → 说不清 → 只参数 → 数据 的次序分类；输入落在快照上
的直接比两边快照的 SQL，不落在快照上的看写它的节点和画布上游
- 口径变了时严格拆分：旧表达式配这次的输入用推演重算的还原和求值算 f_old(x_new)，十进制精确相减，两效应之和
逐位等于总变化；SQL 也变了、用到遮罩列、单位不同、旧叶子没记录的不拆并说明
- 维度归因按维度键配对两边的行、取变化最大的 10 个；度量之和等于输入的值（含合计行那一格）才给占比；截断、
度量列遮罩照实写；遮罩列、日期列、每期都不同的列不当维度
This commit is contained in:
ThearchyHelios
+164
-3
@@ -1,10 +1,171 @@
|
||||
"""运行对比：两次运行按「变化三分法」对照。
|
||||
"""运行对比：同一个工作流的两次运行按「变化三分法」对照——数据变了、口径变了、还是模板 / SQL 变了（engine/compare.py）。
|
||||
|
||||
目前只登记了路由前缀，接口由运行对比的工作包补上。路径挂在 /api/runs/{run_id}/… 下面；
|
||||
这个路由登记在 runs.router 之前，以后加 /api/runs/compare 这类固定路径也不会被 /{run_id} 吃掉。
|
||||
GET /api/runs/{run_id}/compare/base：默认拿哪一次来比，{base_run_id, reason, base}。这次运行是定时任务发起的，取同一
|
||||
定时任务、结束得比它早的最近一次终态运行（reason: same_schedule）；不是、或者同一任务还没有更早的，取同一工作流、
|
||||
同一运行类别、结束得比它早的最近一次终态运行（reason: same_workflow）。只挑封存过的：没封存的拿来也比不了。
|
||||
找不到 404。
|
||||
|
||||
GET /api/runs/{run_id}/compare?base={base_id}：{target, base, summary, template, metrics, queries, notes}，每个字段
|
||||
的形状见 engine/compare.py。两次运行都必须已结束、封存且封存核对一致，封存判定照搬证据接口（api/evidence._sealed）：
|
||||
没封存的运行事件还会变，封存核对不一致的事件被改过，拿它们比出来的结论都不作数，回 409。不是同一个工作流、或者
|
||||
是同一次运行，回 422。
|
||||
|
||||
只读：两次运行都不追加事件、不落工件、不改运行记录；结果只交回给发请求的人。只认各自封存范围内的工件（证据接口的
|
||||
白名单，加上封存的节点产出和指标集里写着的完整值，同推演），遮罩列（查询当时记下的加数据源现在设的）的值不出现在
|
||||
结果里。读工件、比 SQL、重算都在线程里做：快照大的时候不挡住同一进程里别的请求。
|
||||
|
||||
路径挂在 /api/runs/{run_id}/… 下面；这个路由登记在 runs.router 之前，以后加 /api/runs/compare 这类固定路径也不会
|
||||
被 /{run_id} 吃掉。定时任务结束后自动对比（期 4）直接调 default_base、compare_runs。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.api.coded import CodedHTTPException
|
||||
from app.api.evidence import _TERMINAL, RUN_NOT_FOUND, _Masks, _Sealed, _sealed, _sealed_cards
|
||||
from app.api.whatif import _finished_before, _loader, _mask_of, _outputs_before, _returned
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource, Run
|
||||
from app.engine import compare as engine
|
||||
from app.engine import recompute
|
||||
|
||||
router = APIRouter(prefix="/api/runs", tags=["compare"])
|
||||
|
||||
BAD_REQUEST = "compare_bad_request"
|
||||
MISMATCH = "compare_mismatch"
|
||||
UNSEALED = "compare_unsealed"
|
||||
SEAL_BROKEN = "compare_seal_broken"
|
||||
NO_BASE = "compare_no_base"
|
||||
|
||||
BAD_REQUEST_TEXT = "请求格式有误：base 应为作为对比基准的运行 ID"
|
||||
BASE_NOT_FOUND = "作为对比基准的运行记录不存在"
|
||||
SAME_RUN = "不能和同一次运行自己对比，请选择另一次运行作为基准"
|
||||
UNSAVED = "{who}不属于任何已保存的工作流，无法确认两次运行是同一个工作流，不能对比"
|
||||
OTHER_WORKFLOW = "两次运行不属于同一个工作流（「{target}」与「{base}」），不能对比"
|
||||
UNSEALED_TEXT = "{who}尚未结束并封存：对比只用封存的运行记录，运行结束后才能对比"
|
||||
SEAL_BROKEN_TEXT = "{who}的封存核对不一致：封存后有事件被修改、删除或插入，不能用于对比。可在记录页点「核对」查看核对结果"
|
||||
NO_BASE_TEXT = ("没有找到可以对比的上一次运行：需要同一工作流、同一运行类别（定时任务发起的优先找同一定时任务）、"
|
||||
"更早结束并已封存的运行")
|
||||
TARGET = "这次运行"
|
||||
BASE = "作为基准的运行"
|
||||
|
||||
|
||||
@router.get("/{run_id}/compare/base")
|
||||
async def compare_base(run_id: str) -> dict[str, Any]:
|
||||
"""默认拿来对比的那一次运行：{base_run_id, reason: same_schedule | same_workflow, base}。"""
|
||||
found = await default_base(run_id)
|
||||
if found is None:
|
||||
raise CodedHTTPException(404, NO_BASE_TEXT, NO_BASE)
|
||||
run, reason = found
|
||||
return {"base_run_id": run.id, "reason": reason, "base": run_ref(run)}
|
||||
|
||||
|
||||
@router.get("/{run_id}/compare")
|
||||
async def compare_with(run_id: str, base: str | None = None) -> dict[str, Any]:
|
||||
"""这次运行和 base 那一次的对比，形状见 engine/compare.py。"""
|
||||
if base is None or not base.strip():
|
||||
raise CodedHTTPException(422, BAD_REQUEST_TEXT, BAD_REQUEST)
|
||||
return await compare_runs(run_id, base.strip())
|
||||
|
||||
|
||||
async def default_base(run_id: str) -> tuple[Run, str] | None:
|
||||
"""(默认的基准运行, 怎么选的)；找不到返回 None，这次运行本身不存在 404。
|
||||
|
||||
「早」按结束时间比：这次运行还没结束（还在跑、停在审批）的按开始时间。只挑终态、封存过的运行——没封存的拿来
|
||||
对比只会得到 409。定时任务发起的先找同一定时任务（同一个工作流）的；它是这个任务的第一次时，退回同一工作流、
|
||||
同一运行类别。没挂在已保存的工作流上的运行（画布上没保存就运行的图）没有可比的。"""
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, run_id)
|
||||
if run is None:
|
||||
raise CodedHTTPException(404, "运行记录不存在", RUN_NOT_FOUND)
|
||||
if not run.workflow_id:
|
||||
return None
|
||||
before = run.finished_at or run.started_at or run.created_at
|
||||
query = (select(Run)
|
||||
.where(Run.id != run.id, Run.workflow_id == run.workflow_id, Run.status.in_(_TERMINAL),
|
||||
Run.manifest_hash.is_not(None), Run.finished_at.is_not(None), Run.finished_at < before)
|
||||
.order_by(Run.finished_at.desc(), Run.created_at.desc()).limit(1))
|
||||
if run.schedule_id:
|
||||
found = (await session.execute(query.where(Run.schedule_id == run.schedule_id))).scalars().first()
|
||||
if found is not None:
|
||||
return found, "same_schedule"
|
||||
found = (await session.execute(query.where(Run.run_class == run.run_class))).scalars().first()
|
||||
return (found, "same_workflow") if found is not None else None
|
||||
|
||||
|
||||
async def compare_runs(run_id: str, base_id: str) -> dict[str, Any]:
|
||||
"""对比两次运行（run_id 是这次，base_id 是基准）。拒绝时抛 CodedHTTPException：运行不存在 404，同一次运行、
|
||||
不是同一个工作流 422，没结束封存、封存核对不一致 409。"""
|
||||
async with SessionLocal() as session:
|
||||
target = await session.get(Run, run_id)
|
||||
base = await session.get(Run, base_id)
|
||||
kinds = {str(name): kind for name, kind in (await session.execute(select(DataSource.name, DataSource.kind)))}
|
||||
if target is None:
|
||||
raise CodedHTTPException(404, "运行记录不存在", RUN_NOT_FOUND)
|
||||
if base is None:
|
||||
raise CodedHTTPException(404, BASE_NOT_FOUND, RUN_NOT_FOUND)
|
||||
if base.id == target.id:
|
||||
raise CodedHTTPException(422, SAME_RUN, MISMATCH)
|
||||
for run, who in ((target, TARGET), (base, BASE)):
|
||||
if not run.workflow_id:
|
||||
raise CodedHTTPException(422, UNSAVED.format(who=who), MISMATCH)
|
||||
if target.workflow_id != base.workflow_id:
|
||||
raise CodedHTTPException(422, OTHER_WORKFLOW.format(target=target.workflow_name, base=base.workflow_name),
|
||||
MISMATCH)
|
||||
sealed = []
|
||||
for run, who in ((target, TARGET), (base, BASE)):
|
||||
found = await _sealed(run.id)
|
||||
if not found.sealed or found.status not in _TERMINAL:
|
||||
raise CodedHTTPException(409, UNSEALED_TEXT.format(who=who), UNSEALED)
|
||||
if not found.trusted:
|
||||
raise CodedHTTPException(409, SEAL_BROKEN_TEXT.format(who=who), SEAL_BROKEN)
|
||||
sealed.append(found)
|
||||
masks = _Masks()
|
||||
target_side = await _side(target, sealed[0], masks, kinds)
|
||||
base_side = await _side(base, sealed[1], masks, kinds)
|
||||
return await asyncio.to_thread(engine.compare, base_side, target_side)
|
||||
|
||||
|
||||
def run_ref(run: Run) -> dict[str, Any]:
|
||||
"""一次运行的概要（RunRef）：对比结果和默认基准里都用这一个形状。时间带时区（UTC）。"""
|
||||
return {"id": run.id, "workflow_name": run.workflow_name, "version": run.version, "run_class": run.run_class,
|
||||
"started_at": run.started_at.isoformat() if run.started_at else None,
|
||||
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
||||
"schedule_id": run.schedule_id}
|
||||
|
||||
|
||||
async def _side(run: Run, sealed: _Sealed, masks: _Masks, kinds: dict[str, Any]) -> engine.Side:
|
||||
"""一次运行交给对比的东西（engine.compare.Side）：封存范围内的事件、复验过哈希的口径卡、只认封存范围内工件的
|
||||
读法、每份查询快照要遮的列，以及按口径卡那次执行还原输入用的 Sources。"""
|
||||
cards = await asyncio.to_thread(_sealed_cards, sealed)
|
||||
loader = _loader(sealed, cards)
|
||||
# 遮罩要查库（数据源现在设的），在这里每份快照算一遍交进去，对比本身在线程里不碰库。快照先在线程里读好，
|
||||
# 同一个 loader 记着读过的，对比时不再读盘
|
||||
snaps = await asyncio.to_thread(_read_all, loader, list(sealed.queries))
|
||||
masked = {artifact: frozenset(await _mask_of(snaps.get(artifact), info, masks))
|
||||
for artifact, info in sealed.queries.items()}
|
||||
returned = _returned(sealed)
|
||||
|
||||
def sources(started: int) -> recompute.Sources:
|
||||
return recompute.Sources(load=loader, queries=tuple(sealed.queries), outputs=_outputs_before(sealed, started),
|
||||
masked=masked, returned=returned, finished=_finished_before(sealed, started))
|
||||
|
||||
return engine.Side(run=run_ref(run), graph=sealed.graph, events=sealed.events, cards=cards, load=loader,
|
||||
masked=masked, sources=sources, dialects=kinds, inputs=dict(run.input or {}))
|
||||
|
||||
|
||||
def _read_all(loader: Callable[[str], Any], artifacts: list[str]) -> dict[str, Any]:
|
||||
"""读出这几份查询快照：读不出来、被改过的记 None（遮罩照样按台账记的数据源算）。"""
|
||||
out: dict[str, Any] = {}
|
||||
for artifact in artifacts:
|
||||
try:
|
||||
content = loader(artifact)
|
||||
except ValueError:
|
||||
content = None
|
||||
out[artifact] = content if isinstance(content, dict) else None
|
||||
return out
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+211
@@ -0,0 +1,211 @@
|
||||
"""运行对比的虚构运行（tests/test_compare_engine.py、tests/test_compare_api.py 共用）。
|
||||
|
||||
两次周报运行照真实运行的事件形状手写（区域、金额、日期都是编的）：取数节点 fetch 查一条按区域汇总的 SQL，Agent 节点
|
||||
bot 自己再查一次、把各区域的金额作为数组字段 amounts 交给口径卡，口径卡算几个指标。工件按内容寻址放在字典里，
|
||||
接口测试把它们原样落进工件库、事件落进运行记录。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from app.core.artifact_store import canonical_json, content_hash
|
||||
from app.engine import compare, recompute
|
||||
from app.engine.evidence import render_number
|
||||
from app.engine.expressions import parse_expression, substitute
|
||||
|
||||
TOOL = "db_query__shop"
|
||||
WEEK36 = ("2026-08-31", "2026-09-06")
|
||||
WEEK37 = ("2026-09-07", "2026-09-13")
|
||||
COLUMNS = ["region", "gmv", "refund"]
|
||||
TYPES = {"region": "text", "gmv": "number", "refund": "number"}
|
||||
BASE_ROWS = [["东区", 1200.5, 100.0], ["南区", 800.0, 50.0], ["西区", 600.0, 0.0]]
|
||||
NEXT_ROWS = [["东区", 1500.5, 120.0], ["南区", 700.0, 40.0], ["西区", 600.0, 10.0]]
|
||||
EAST = "cell(nodes.fetch, 0, 'gmv')"
|
||||
REFUND = "cell(nodes.fetch, 0, 'refund')"
|
||||
|
||||
|
||||
def region_sql(days=WEEK36, extra=""):
|
||||
return ("SELECT region, SUM(amount) AS gmv, SUM(refund) AS refund FROM orders "
|
||||
f"WHERE day BETWEEN '{days[0]}' AND '{days[1]}'{extra} GROUP BY region ORDER BY region")
|
||||
|
||||
|
||||
def node(nid, ntype, label, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label, "config": config}}
|
||||
|
||||
|
||||
def chain(*nodes):
|
||||
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||||
|
||||
|
||||
def weekly_graph(sql=None, *, between=(), card_label="周报口径卡"):
|
||||
return chain(
|
||||
node("start", "input", "开始"),
|
||||
node("fetch", "tool", "区域取数", tool=TOOL, args={"sql": sql or region_sql()}),
|
||||
node("bot", "agent", "区域分析", prompt="按区域列出金额", tools=[TOOL], cite_fields=True),
|
||||
*between,
|
||||
node("card", "metrics", card_label, caliber="周报口径", caliber_version="v1"),
|
||||
node("write", "report", "写周报"),
|
||||
node("out", "output", "成果"),
|
||||
)
|
||||
|
||||
|
||||
class Fake:
|
||||
"""一次虚构的运行：工件按内容寻址放在字典里，事件照真实运行的形状写（tool.start / tool.end / node.finished
|
||||
的证据台账、口径卡的 metric_set 条目、节点产出工件）。"""
|
||||
|
||||
def __init__(self, run_id, *, graph=None, version=3, inputs=None, masked=None, source="shop"):
|
||||
self.id = run_id
|
||||
self.source = source
|
||||
self.graph = graph or weekly_graph()
|
||||
self.version = version
|
||||
self.inputs = inputs or {}
|
||||
self.masked = masked or {}
|
||||
self.store: dict[str, object] = {}
|
||||
self.tampered: set[str] = set()
|
||||
self.events: list[tuple[int, str, str | None, dict]] = []
|
||||
self.cards: list[tuple[str, str, dict]] = []
|
||||
self.queries: list[str] = []
|
||||
self.elapsed = 0
|
||||
self.add("run.started", None, {"nodes": len(self.graph["nodes"])})
|
||||
|
||||
def put(self, obj):
|
||||
text = canonical_json(obj)
|
||||
artifact = content_hash(text)
|
||||
self.store[artifact] = json.loads(text)
|
||||
return artifact
|
||||
|
||||
def load(self, artifact):
|
||||
if artifact in self.tampered:
|
||||
raise ValueError("工件内容与哈希不符，疑似被篡改")
|
||||
return self.store.get(artifact)
|
||||
|
||||
def add(self, etype, node_id, data):
|
||||
self.events.append((len(self.events) + 1, etype, node_id, data))
|
||||
|
||||
def snapshot(self, sql, columns, rows, *, truncated=False, types=None, masks=None):
|
||||
self.elapsed += 1 # 真实快照带着耗时：同一条 SQL 查两次也是两件工件
|
||||
snap = {"columns": columns, "rows": rows, "row_count": len(rows), "truncated": truncated,
|
||||
"elapsed_ms": self.elapsed, "sql": sql, "source": self.source,
|
||||
**({"column_types": types} if types else {}), **({"mask_columns": masks} if masks else {})}
|
||||
artifact = self.put(snap)
|
||||
self.queries.append(artifact)
|
||||
return artifact
|
||||
|
||||
def entry(self, node_id, artifact, columns, rows, *, tool=TOOL, call_id=None, truncated=False):
|
||||
return {"kind": "query", "node_id": node_id, "exec": 1, "artifact": artifact, "tool": tool,
|
||||
"source": self.source,
|
||||
"columns": columns, "rows": len(rows), "truncated": truncated,
|
||||
**({"call_id": call_id} if call_id else {})}
|
||||
|
||||
def tool(self, node_id, sql, columns=COLUMNS, rows=BASE_ROWS, *, tool=TOOL, truncated=False, types=TYPES,
|
||||
masks=None):
|
||||
"""调用工具节点查一次库。masks 是查询当时数据源设的遮罩（记进快照的 mask_columns）。"""
|
||||
artifact = self.snapshot(sql, columns, rows, truncated=truncated, types=types, masks=masks)
|
||||
output = self.put({"columns": columns, "rows": rows, "sql": sql, "source": self.source, "artifact": artifact})
|
||||
self.add("node.started", node_id, {"node_type": "tool"})
|
||||
self.add("tool.start", node_id, {"tool": tool, "args": {"sql": sql}})
|
||||
self.add("tool.end", node_id, {"tool": tool, "duration_ms": 3, "query_artifact": artifact})
|
||||
self.add("node.finished", node_id, {"duration_ms": 4, "attempt": 1, "artifact": output, "evidence": [
|
||||
self.entry(node_id, artifact, columns, rows, tool=tool, truncated=truncated)]})
|
||||
return artifact
|
||||
|
||||
def agent(self, node_id, sql, columns=COLUMNS, rows=BASE_ROWS, *, types=TYPES, truncated=False, masks=None,
|
||||
verify=()):
|
||||
"""Agent 节点自己查一次库，按出处交回数组字段 amounts（快照 gmv 列的全部行）。verify 是同一节点里核查类的
|
||||
查询（role: verify，不进台账）。"""
|
||||
artifact = self.snapshot(sql, columns, rows, truncated=truncated, types=types, masks=masks)
|
||||
last = len(rows) - 1
|
||||
gmv = columns.index("gmv")
|
||||
data = {"amounts": [r[gmv] for r in rows]}
|
||||
evidence = {"amounts": {"ref": f"Q1.r0-{last}.gmv", "call": "Q1", "artifact": artifact,
|
||||
"locator": {"rows": [0, last], "column": "gmv"}, "status": "verified"}}
|
||||
output = self.put({"text": "按区域列出了金额", "data": data, "data_evidence": evidence})
|
||||
self.add("node.started", node_id, {"node_type": "agent"})
|
||||
self.add("tool.start", node_id, {"tool": TOOL, "args": {"sql": sql}, "call_id": "c1"})
|
||||
self.add("tool.end", node_id, {"tool": TOOL, "call_id": "c1", "duration_ms": 5, "query_artifact": artifact})
|
||||
for i, extra in enumerate(verify):
|
||||
check = self.snapshot(extra, ["n"], [[1]])
|
||||
self.add("tool.end", node_id, {"tool": TOOL, "call_id": f"v{i}", "role": "verify", "query_artifact": check})
|
||||
self.add("node.finished", node_id, {"duration_ms": 9, "attempt": 1, "artifact": output, "evidence": [
|
||||
self.entry(node_id, artifact, columns, rows, call_id="c1", truncated=truncated)]})
|
||||
return artifact
|
||||
|
||||
def code(self, node_id, value, var):
|
||||
"""沙箱代码节点算出一个数写进变量：它不查库，对不上具体的查询。"""
|
||||
output = self.put({var: value})
|
||||
self.add("node.started", node_id, {"node_type": "code"})
|
||||
self.add("node.finished", node_id, {"duration_ms": 2, "attempt": 1, "artifact": output})
|
||||
|
||||
def card(self, metrics, *, node_id="card", caliber="周报口径", version="v1", runs=1):
|
||||
card = {"kind": "metric_set", "caliber": caliber, "caliber_version": version, "metrics": metrics}
|
||||
artifact = self.put(card)
|
||||
for _ in range(runs):
|
||||
self.add("node.started", node_id, {"node_type": "metrics"})
|
||||
self.add("node.finished", node_id, {"duration_ms": 1, "attempt": 1, "artifact": self.put({**card, "text": "…"}),
|
||||
"evidence": [{"kind": "metric_set", "node_id": node_id, "exec": 1,
|
||||
"artifact": artifact, "caliber": caliber,
|
||||
"version": version}]})
|
||||
self.cards.append((node_id, artifact, card))
|
||||
return artifact
|
||||
|
||||
def side(self):
|
||||
def sources(started):
|
||||
outputs = {}
|
||||
for seq, etype, nid, data in self.events:
|
||||
if seq < started and etype == "node.finished" and data.get("artifact"):
|
||||
outputs[nid] = data["artifact"]
|
||||
return recompute.Sources(load=self.load, queries=tuple(self.queries), outputs=outputs, masked=self.masked)
|
||||
|
||||
run = {"id": self.id, "workflow_name": "区域周报", "version": self.version, "run_class": "formal",
|
||||
"started_at": "2026-09-07T01:00:00+00:00", "finished_at": "2026-09-07T01:02:00+00:00",
|
||||
"schedule_id": None}
|
||||
return compare.Side(run=run, graph=self.graph, events=self.events, cards=self.cards, load=self.load,
|
||||
masked=self.masked, sources=sources, dialects={self.source: "sqlite"}, inputs=self.inputs)
|
||||
|
||||
|
||||
def cell_in(row, column, value, artifact, node_id="fetch"):
|
||||
return {"path": f"cell(nodes.{node_id}, {row}, '{column}')", "value": value, "node_id": node_id, "via": "tool_cell",
|
||||
"locator": {"row": row, "column": column}, "artifact": artifact,
|
||||
"eid": f"ev:cell:{artifact[:8]}:{row}:{column}", "status": "verified"}
|
||||
|
||||
|
||||
def amounts_in(artifact, rows):
|
||||
last = len(rows) - 1
|
||||
return {"path": "vars.kpi.amounts", "value": f"[{len(rows)} 项]", "node_id": "bot", "via": "agent_field",
|
||||
"field": "amounts", "ref": f"Q1.r0-{last}.gmv", "artifact": artifact,
|
||||
"locator": {"rows": [0, last], "column": "gmv"}, "eid": f"ev:rows:{artifact[:8]}", "status": "verified"}
|
||||
|
||||
|
||||
def scalar_in(path, value, node_id, via):
|
||||
return {"path": path, "value": value, "node_id": node_id, "via": via, "field": path.split(".", 2)[-1],
|
||||
"status": "ok" if value is not None else "missing"}
|
||||
|
||||
|
||||
def metric(mid, expression, value, inputs, *, name=None, unit="元", decimals=1, fmt="thousands"):
|
||||
tree = parse_expression(expression)[0]
|
||||
scalars = {i["path"]: i["value"] for i in inputs if not str(i["value"]).endswith(" 项]")}
|
||||
return {"id": mid, "name": name or mid, "unit": unit, "value": value, "expression": expression,
|
||||
"decimals": decimals, "format": fmt,
|
||||
"rendered": render_number(value, unit=unit, decimals=decimals, fmt=fmt),
|
||||
"inputs": inputs, "substituted": substitute(tree, scalars), "recompute_ok": True,
|
||||
"status": "ok" if value is not None else "missing_input"}
|
||||
|
||||
|
||||
def week(run_id, rows=BASE_ROWS, *, sql=None, metrics=None, graph=None, version="v1", bot_rows=None, **kw):
|
||||
"""一次周报运行：fetch 和 bot 各查一次，口径卡算 east（东区金额）和 total（各区域合计）。
|
||||
metrics(fetch, bot, rows) 返回口径卡的指标，不给就是这两个。"""
|
||||
run = Fake(run_id, graph=graph or weekly_graph(sql), **kw)
|
||||
sql = sql or region_sql()
|
||||
fetch = run.tool("fetch", sql, rows=rows)
|
||||
bot_rows = bot_rows if bot_rows is not None else rows
|
||||
bot = run.agent("bot", sql, rows=bot_rows)
|
||||
if metrics is None:
|
||||
metrics = default_metrics
|
||||
run.card(metrics(fetch, bot, rows, bot_rows), version=version)
|
||||
return run
|
||||
|
||||
|
||||
def default_metrics(fetch, bot, rows, bot_rows):
|
||||
total = sum(r[1] for r in bot_rows)
|
||||
return [metric("east", EAST, rows[0][1], [cell_in(0, "gmv", rows[0][1], fetch)], name="东区金额"),
|
||||
metric("total", "sum(vars.kpi.amounts)", total, [amounts_in(bot, bot_rows)], name="各区域合计")]
|
||||
@@ -0,0 +1,387 @@
|
||||
"""运行对比的接口：GET /api/runs/{run_id}/compare/base 和 GET /api/runs/{run_id}/compare?base=。
|
||||
|
||||
两次周报运行用 tests/fixtures/compare_runs.py 的虚构运行：工件落进工件库，事件落进运行记录，按 runner 的算法封存
|
||||
（不连库、不调模型）。另有一组真跑：虚构的 SQLite 库在两次运行之间改了数，再一次代进去的日期参数不同。守的是：
|
||||
- 两次都结束、封存且核对一致才比：没封存、还在运行、封存后事件被改过都回 409，说清是哪一次；不是同一个工作流、
|
||||
同一次运行回 422；运行不存在 404
|
||||
- 对比只读：两次运行的 last_seq、事件、工件引用、工件文件一样不多
|
||||
- 遮罩列（查询当时记下的、数据源现在设的）的值不出现在结果里
|
||||
- 默认基准：同一定时任务优先，否则同一工作流、同一运行类别；只挑更早结束、封存过的终态运行
|
||||
- 真跑：同一条 SQL 数据变了是 data_only；只是代进去的日期不同是 params_only，图一样
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app.core.artifact_store import manifest_hash, put_json
|
||||
from app.core.config import settings
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact, DataSource, Run, RunEvent, Workflow
|
||||
from app.engine.runner import run_manager, verify_manifest
|
||||
from app.main import app
|
||||
from tests.fixtures.compare_runs import (
|
||||
BASE_ROWS,
|
||||
EAST,
|
||||
NEXT_ROWS,
|
||||
Fake,
|
||||
amounts_in,
|
||||
cell_in,
|
||||
chain,
|
||||
metric,
|
||||
node,
|
||||
week,
|
||||
)
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
SOURCE = "cmp_shop"
|
||||
T0 = datetime(2026, 9, 7, 1, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
async def new_workflow(name="区域周报") -> str:
|
||||
async with SessionLocal() as session:
|
||||
workflow = Workflow(name=name)
|
||||
session.add(workflow)
|
||||
await session.commit()
|
||||
return workflow.id
|
||||
|
||||
|
||||
async def persist(fake: Fake, workflow_id: str | None, *, finished=T0, status="succeeded", run_class="formal",
|
||||
schedule_id=None, seal=True) -> str:
|
||||
"""把一次虚构运行落库：工件进工件库（内容寻址，id 和虚构时一样），事件进运行记录，按 runner 的算法封存。"""
|
||||
for content in fake.store.values():
|
||||
await put_json(content, kind="compare_fixture")
|
||||
async with SessionLocal() as session:
|
||||
run = Run(workflow_id=workflow_id, workflow_name="区域周报", status=status, graph=fake.graph,
|
||||
input=dict(fake.inputs), run_class=run_class, version=fake.version, schedule_id=schedule_id,
|
||||
started_at=finished - timedelta(minutes=2), finished_at=finished if status != "running" else None,
|
||||
agent_limits={})
|
||||
session.add(run)
|
||||
await session.flush()
|
||||
rows = []
|
||||
for seq, etype, node_id, data in fake.events:
|
||||
session.add(RunEvent(run_id=run.id, seq=seq, type=etype, node_id=node_id, ts=float(seq), data=data))
|
||||
rows.append((seq, etype, node_id, data))
|
||||
run.last_seq = len(rows)
|
||||
if seal:
|
||||
run.manifest_hash, run.manifest_seq = manifest_hash(rows), rows[-1][0]
|
||||
await session.commit()
|
||||
return run.id
|
||||
|
||||
|
||||
async def pair(base=None, target=None, **kw) -> tuple[str, str, str]:
|
||||
"""(工作流, 基准运行, 这次运行)：同一个工作流的两次周报，这次的数变了。"""
|
||||
workflow = await new_workflow()
|
||||
base_id = await persist(base or week("r1", source=SOURCE), workflow, finished=T0)
|
||||
target_id = await persist(target or week("r2", NEXT_ROWS, source=SOURCE), workflow,
|
||||
finished=T0 + timedelta(days=7), **kw)
|
||||
return workflow, base_id, target_id
|
||||
|
||||
|
||||
async def footprint(*run_ids: str) -> tuple:
|
||||
"""几次运行留在库里、盘上的痕迹：各自的 last_seq、封存、事件数，加上全部工件引用行数和工件文件数。"""
|
||||
async with SessionLocal() as session:
|
||||
runs = []
|
||||
for run_id in run_ids:
|
||||
run = await session.get(Run, run_id)
|
||||
count = await session.scalar(select(func.count()).select_from(RunEvent).where(RunEvent.run_id == run_id))
|
||||
runs.append((run.last_seq, run.manifest_hash, run.manifest_seq, run.status, count))
|
||||
artifacts = await session.scalar(select(func.count()).select_from(Artifact))
|
||||
files = sum(1 for _ in (settings.data_dir / "artifacts").rglob("*.json"))
|
||||
return runs, artifacts, files
|
||||
|
||||
|
||||
async def get(client, path: str, status: int = 200) -> dict:
|
||||
resp = await client.get(path)
|
||||
assert resp.status_code == status, resp.text
|
||||
return resp.json()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 对比
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_compares_two_sealed_runs(client):
|
||||
_, base_id, target_id = await pair()
|
||||
body = await get(client, f"/api/runs/{target_id}/compare?base={base_id}")
|
||||
assert body["target"]["id"] == target_id and body["base"]["id"] == base_id
|
||||
assert set(body["target"]) == {"id", "workflow_name", "version", "run_class", "started_at", "finished_at",
|
||||
"schedule_id"}
|
||||
assert body["target"]["finished_at"].endswith("+00:00") and body["base"]["run_class"] == "formal"
|
||||
assert body["summary"]["data_only"] == 2 and sum(body["summary"].values()) == 2
|
||||
east = next(m for m in body["metrics"] if m["metric_id"] == "east")
|
||||
assert (east["classification"], east["base"], east["target"], east["delta_rendered"]) == (
|
||||
"data_only", 1200.5, 1500.5, "+300.0元")
|
||||
total = next(m for m in body["metrics"] if m["metric_id"] == "total")
|
||||
assert [r["share"] for r in total["attribution"]["rows"]] == [1.5, -0.5, 0.0]
|
||||
assert [(q["node_id"], q["structure"]) for q in body["queries"]] == [("fetch", "same"), ("bot", "same")]
|
||||
assert body["template"] == {"base_version": 3, "target_version": 3, "same_graph": True, "nodes": []}
|
||||
assert body["notes"] == []
|
||||
|
||||
|
||||
async def test_comparing_writes_nothing(client):
|
||||
_, base_id, target_id = await pair()
|
||||
before = await footprint(base_id, target_id)
|
||||
await get(client, f"/api/runs/{target_id}/compare?base={base_id}")
|
||||
await get(client, f"/api/runs/{target_id}/compare/base")
|
||||
assert await footprint(base_id, target_id) == before
|
||||
for run_id in (base_id, target_id):
|
||||
assert (await verify_manifest(run_id))["ok"] is True
|
||||
|
||||
|
||||
async def test_unsealed_runs_are_refused(client):
|
||||
workflow = await new_workflow()
|
||||
base_id = await persist(week("r1", source=SOURCE), workflow)
|
||||
target_id = await persist(week("r2", NEXT_ROWS, source=SOURCE), workflow, finished=T0 + timedelta(days=7),
|
||||
seal=False)
|
||||
body = await get(client, f"/api/runs/{target_id}/compare?base={base_id}", 409)
|
||||
assert body["code"] == "compare_unsealed" and body["detail"].startswith("这次运行")
|
||||
body = await get(client, f"/api/runs/{base_id}/compare?base={target_id}", 409)
|
||||
assert body["code"] == "compare_unsealed" and body["detail"].startswith("作为基准的运行")
|
||||
# 封存过、又被接着运行（还在运行）的：封存范围之后还会变，同样不比
|
||||
running = await persist(week("r3", NEXT_ROWS, source=SOURCE), workflow, status="running")
|
||||
body = await get(client, f"/api/runs/{running}/compare?base={base_id}", 409)
|
||||
assert body["code"] == "compare_unsealed"
|
||||
|
||||
|
||||
async def test_a_broken_seal_is_refused(client):
|
||||
_, base_id, target_id = await pair()
|
||||
async with SessionLocal() as session:
|
||||
event = (await session.execute(select(RunEvent).where(
|
||||
RunEvent.run_id == base_id, RunEvent.type == "tool.end"))).scalars().first()
|
||||
event.data = {**event.data, "duration_ms": 999}
|
||||
await session.commit()
|
||||
body = await get(client, f"/api/runs/{target_id}/compare?base={base_id}", 409)
|
||||
assert body["code"] == "compare_seal_broken" and body["detail"].startswith("作为基准的运行")
|
||||
assert "封存核对不一致" in body["detail"]
|
||||
|
||||
|
||||
async def test_mismatched_runs_are_refused(client):
|
||||
_, base_id, target_id = await pair()
|
||||
other = await persist(week("r9", source=SOURCE), await new_workflow("另一个周报"))
|
||||
body = await get(client, f"/api/runs/{target_id}/compare?base={other}", 422)
|
||||
assert body["code"] == "compare_mismatch" and "同一个工作流" in body["detail"]
|
||||
body = await get(client, f"/api/runs/{target_id}/compare?base={target_id}", 422)
|
||||
assert body["code"] == "compare_mismatch" and "同一次运行" in body["detail"]
|
||||
unsaved = await persist(week("r8", source=SOURCE), None)
|
||||
body = await get(client, f"/api/runs/{unsaved}/compare?base={base_id}", 422)
|
||||
assert body["code"] == "compare_mismatch"
|
||||
body = await get(client, f"/api/runs/{target_id}/compare", 422)
|
||||
assert body["code"] == "compare_bad_request"
|
||||
await get(client, f"/api/runs/no-such-run/compare?base={base_id}", 404)
|
||||
body = await get(client, f"/api/runs/{target_id}/compare?base=no-such-run", 404)
|
||||
assert "基准" in body["detail"]
|
||||
|
||||
|
||||
async def test_masked_columns_never_leave_the_server(client):
|
||||
"""快照里记着查询当时遮了 refund，数据源现在又遮了 manager：两样的值都不出现在结果里。"""
|
||||
columns = ["region", "manager", "gmv", "refund"]
|
||||
types = {"region": "text", "manager": "text", "gmv": "number", "refund": "number"}
|
||||
sql = "SELECT region, manager, SUM(amount) AS gmv, SUM(refund) AS refund FROM orders GROUP BY region, manager"
|
||||
|
||||
def make(run_id, rows, expression, version):
|
||||
run = Fake(run_id, source=SOURCE)
|
||||
fetch = run.tool("fetch", sql, columns, rows, types=types, masks=["refund"])
|
||||
bot = run.agent("bot", sql, columns, rows, types=types, masks=["refund"])
|
||||
inputs = [cell_in(0, "gmv", rows[0][2], fetch)]
|
||||
value = rows[0][2]
|
||||
if "refund" in expression:
|
||||
inputs.append(cell_in(0, "refund", rows[0][3], fetch))
|
||||
value = rows[0][2] - rows[0][3]
|
||||
run.card([metric("net", expression, value, inputs),
|
||||
metric("total", "sum(vars.kpi.amounts)", sum(r[2] for r in rows), [amounts_in(bot, rows)])],
|
||||
version=version)
|
||||
return run
|
||||
|
||||
async with SessionLocal() as session:
|
||||
session.add(DataSource(name=SOURCE, kind="sqlite", database=":memory:", options={"mask_columns": ["manager"]}))
|
||||
await session.commit()
|
||||
try:
|
||||
base = make("r1", [["东区", "经理甲", 1200.5, 101.25], ["南区", "经理乙", 800.0, 52.75]], EAST, "v1")
|
||||
target = make("r2", [["东区", "经理甲", 1500.5, 133.5], ["南区", "经理乙", 700.0, 47.25]],
|
||||
f"{EAST} - cell(nodes.fetch, 0, 'refund')", "v2")
|
||||
_, base_id, target_id = await pair(base, target)
|
||||
resp = await client.get(f"/api/runs/{target_id}/compare?base={base_id}")
|
||||
assert resp.status_code == 200, resp.text
|
||||
for secret in ("经理甲", "经理乙", "101.25", "52.75", "133.5", "47.25"):
|
||||
assert secret not in resp.text, secret
|
||||
body = resp.json()
|
||||
net = next(m for m in body["metrics"] if m["metric_id"] == "net")
|
||||
refund = next(i for i in net["inputs"] if "refund" in i["path"])
|
||||
assert refund["masked"] is True and net["decomposition"] is None and "遮罩" in net["decomposition_note"]
|
||||
total = next(m for m in body["metrics"] if m["metric_id"] == "total")
|
||||
assert total["attribution"]["columns"] == ["region"]
|
||||
finally:
|
||||
await drop_source(name=SOURCE)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 默认基准
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def bare(workflow_id, finished, *, status="succeeded", run_class="formal", schedule_id=None, sealed=True) -> str:
|
||||
"""只为挑基准用的运行记录：挑基准只看运行记录上的几列。"""
|
||||
async with SessionLocal() as session:
|
||||
run = Run(workflow_id=workflow_id, workflow_name="区域周报", status=status, run_class=run_class,
|
||||
schedule_id=schedule_id, started_at=finished - timedelta(minutes=2),
|
||||
finished_at=finished if status in ("succeeded", "failed", "cancelled") else None,
|
||||
manifest_hash="ab" * 32 if sealed else None, manifest_seq=1 if sealed else None)
|
||||
session.add(run)
|
||||
await session.commit()
|
||||
return run.id
|
||||
|
||||
|
||||
async def test_the_default_base_prefers_the_same_schedule(client):
|
||||
workflow = await new_workflow()
|
||||
day = timedelta(days=1)
|
||||
scheduled = await bare(workflow, T0, schedule_id="sched-weekly")
|
||||
manual = await bare(workflow, T0 + day) # 更近，但不是同一个定时任务
|
||||
current = await bare(workflow, T0 + 2 * day, schedule_id="sched-weekly")
|
||||
body = await get(client, f"/api/runs/{current}/compare/base")
|
||||
assert (body["base_run_id"], body["reason"]) == (scheduled, "same_schedule")
|
||||
assert body["base"]["id"] == scheduled and body["base"]["schedule_id"] == "sched-weekly"
|
||||
# 不是定时任务发起的：同一工作流、同一运行类别里最近的一次（定时的也算）
|
||||
later = await bare(workflow, T0 + 3 * day)
|
||||
body = await get(client, f"/api/runs/{later}/compare/base")
|
||||
assert (body["base_run_id"], body["reason"]) == (current, "same_workflow")
|
||||
# 定时任务的第一次：退回同一工作流、同一运行类别
|
||||
first = await bare(workflow, T0 + 4 * day, schedule_id="sched-daily")
|
||||
body = await get(client, f"/api/runs/{first}/compare/base")
|
||||
assert (body["base_run_id"], body["reason"]) == (later, "same_workflow")
|
||||
assert manual != body["base_run_id"]
|
||||
|
||||
|
||||
async def test_the_default_base_skips_what_cannot_be_compared(client):
|
||||
workflow = await new_workflow()
|
||||
day = timedelta(days=1)
|
||||
good = await bare(workflow, T0)
|
||||
await bare(workflow, T0 + day, sealed=False) # 没封存
|
||||
await bare(workflow, T0 + 2 * day, status="running") # 还在运行
|
||||
await bare(workflow, T0 + 3 * day, run_class="exploratory") # 别的运行类别
|
||||
await bare(await new_workflow("另一个周报"), T0 + 4 * day) # 别的工作流
|
||||
current = await bare(workflow, T0 + 5 * day)
|
||||
await bare(workflow, T0 + 6 * day) # 比它晚结束
|
||||
body = await get(client, f"/api/runs/{current}/compare/base")
|
||||
assert (body["base_run_id"], body["reason"]) == (good, "same_workflow")
|
||||
# 失败、取消的终态运行也算（封存过）
|
||||
failed = await bare(workflow, T0 + 5 * day - timedelta(hours=1), status="failed")
|
||||
assert (await get(client, f"/api/runs/{current}/compare/base"))["base_run_id"] == failed
|
||||
# 第一次运行、没保存的工作流：没有可比的
|
||||
body = await get(client, f"/api/runs/{good}/compare/base", 404)
|
||||
assert body["code"] == "compare_no_base" and "上一次运行" in body["detail"]
|
||||
await get(client, f"/api/runs/{await bare(None, T0)}/compare/base", 404)
|
||||
await get(client, "/api/runs/no-such-run/compare/base", 404)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 真跑：虚构的 SQLite 库，两次运行之间改了数、换了日期参数
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
E2E_SOURCE = "cmp_e2e"
|
||||
E2E_SQL = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE day BETWEEN '{{ input.start }}' AND '{{ input.end }}' "
|
||||
"GROUP BY region ORDER BY region")
|
||||
WEEK36_INPUT = {"start": "2026-08-31", "end": "2026-09-06"}
|
||||
WEEK37_INPUT = {"start": "2026-09-07", "end": "2026-09-13"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def engine_up():
|
||||
await run_manager.setup()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def orders(tmp_path):
|
||||
path = tmp_path / "orders.db"
|
||||
db = sqlite3.connect(path)
|
||||
db.executescript("CREATE TABLE orders (id INTEGER PRIMARY KEY, day TEXT, region TEXT, amount REAL);")
|
||||
db.executemany("INSERT INTO orders (day, region, amount) VALUES (?, ?, ?)", [
|
||||
("2026-09-01", "东区", 1200.5), ("2026-09-02", "南区", 800.0), ("2026-09-03", "西区", 600.0),
|
||||
("2026-09-08", "东区", 900.0), ("2026-09-09", "南区", 650.0), ("2026-09-10", "西区", 640.0)])
|
||||
db.commit()
|
||||
db.close()
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=E2E_SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
source_id = row.id
|
||||
await engines.invalidate(source_id)
|
||||
yield path, source_id
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
def e2e_graph():
|
||||
return chain(
|
||||
node("start", "input", "开始"),
|
||||
node("fetch", "tool", "区域取数", tool=f"db_query__{E2E_SOURCE}", args={"sql": E2E_SQL}),
|
||||
node("card", "metrics", "周报口径卡", caliber="周报口径", caliber_version="v1", metrics=[
|
||||
{"id": "east", "name": "东区金额", "unit": "元", "decimals": 1, "format": "thousands",
|
||||
"expression": "cell(nodes.fetch, 0, 'gmv')"}]),
|
||||
node("out", "output", "成果", fields=[{"name": "指标", "value": "{{ nodes.card.text }}"}]),
|
||||
)
|
||||
|
||||
|
||||
async def finished_run(workflow_id: str, inputs: dict) -> str:
|
||||
run = await run_manager.start(graph=e2e_graph(), input_payload=inputs, workflow_id=workflow_id,
|
||||
workflow_name="区域周报")
|
||||
for _ in range(400):
|
||||
await asyncio.sleep(0.05)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run.id)
|
||||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||||
assert row.status == "succeeded", row.error
|
||||
return run.id
|
||||
raise AssertionError(f"运行 {run.id} 没有结束：{row.status}")
|
||||
|
||||
|
||||
async def test_real_runs_split_data_and_parameter_changes(client, engine_up, orders):
|
||||
path, source_id = orders
|
||||
workflow = await new_workflow()
|
||||
first = await finished_run(workflow, WEEK36_INPUT)
|
||||
db = sqlite3.connect(path)
|
||||
db.execute("UPDATE orders SET amount = 1500.5 WHERE day = '2026-09-01'")
|
||||
db.commit()
|
||||
db.close()
|
||||
await engines.invalidate(source_id)
|
||||
second = await finished_run(workflow, WEEK36_INPUT)
|
||||
third = await finished_run(workflow, WEEK37_INPUT)
|
||||
|
||||
body = await get(client, f"/api/runs/{second}/compare?base={first}")
|
||||
[east] = body["metrics"]
|
||||
assert (east["classification"], east["base"], east["target"], east["delta"]) == ("data_only", 1200.5, 1500.5, 300.0)
|
||||
[fetch] = body["queries"]
|
||||
assert (fetch["node_id"], fetch["structure"], fetch["base_rows"], fetch["target_rows"]) == ("fetch", "same", 3, 3)
|
||||
assert east["inputs"][0]["base_artifact"] == fetch["base_artifact"]
|
||||
# 东区金额只是三个区域里的一格：各区域的变化照列，占比为空
|
||||
assert [(r["key"], r["delta"]) for r in east["attribution"]["rows"]] == [
|
||||
(["东区"], 300.0), (["南区"], 0), (["西区"], 0)]
|
||||
|
||||
body = await get(client, f"/api/runs/{third}/compare?base={second}")
|
||||
[east] = body["metrics"]
|
||||
assert east["classification"] == "params_only" and east["target"] == 900.0
|
||||
[fetch] = body["queries"]
|
||||
assert fetch["structure"] == "params_only"
|
||||
assert fetch["literals"] == [{"base": "'2026-08-31'", "target": "'2026-09-07'"},
|
||||
{"base": "'2026-09-06'", "target": "'2026-09-13'"}]
|
||||
assert body["template"]["same_graph"] is True
|
||||
assert any("start" in n for n in body["notes"]), body["notes"]
|
||||
|
||||
body = await get(client, f"/api/runs/{third}/compare/base")
|
||||
assert (body["base_run_id"], body["reason"]) == (second, "same_workflow")
|
||||
json.dumps(body, allow_nan=False)
|
||||
@@ -0,0 +1,558 @@
|
||||
"""运行对比的纯函数（engine/compare.py）：同一个工作流的两次运行，每个指标的变化拆成数据、口径、模板 / SQL。
|
||||
|
||||
两次运行都是照真实运行的事件形状手写的虚构数据（区域、金额、日期都是编的）：取数节点 fetch 查一条按区域汇总的
|
||||
SQL，Agent 节点 bot 自己再查一次、把各区域的金额作为数组字段交给口径卡，口径卡算几个指标。工件放在字典里。守的是：
|
||||
- 只数据变（SQL 一字不差、结果变了）是 data_only；只改日期字面量是 params_only；SQL 结构变了是 sql_changed、不拆
|
||||
- 口径变了（表达式、口径版本）是 caliber_and_data：旧表达式的叶子路径在这次的输入里都取得到时严格拆分，
|
||||
口径效应 + 数据效应 = 总变化（逐位相等）；取不到就不拆，说明缺的是哪一个
|
||||
- 维度归因：按维度键配对两边的行，按变化的绝对值取前 10；度量列之和等于输入的值（两边都等）才给占比，占比之和
|
||||
为 1；对不上时占比为空并说明；截断的快照不归因，照实写
|
||||
- 遮罩列的值绝不出现在结果里；用到遮罩列的指标不拆分
|
||||
- 模板对比：新增、删除的节点，配置有变化的节点和配置项名
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
from app.engine import compare
|
||||
from tests.fixtures.compare_runs import (
|
||||
BASE_ROWS,
|
||||
EAST,
|
||||
NEXT_ROWS,
|
||||
REFUND,
|
||||
WEEK37,
|
||||
Fake,
|
||||
amounts_in,
|
||||
cell_in,
|
||||
default_metrics,
|
||||
metric,
|
||||
node,
|
||||
region_sql,
|
||||
scalar_in,
|
||||
week,
|
||||
weekly_graph,
|
||||
)
|
||||
|
||||
|
||||
def run_compare(base, target):
|
||||
result = compare.compare(base.side(), target.side())
|
||||
json.dumps(result, allow_nan=False) # 结果要能原样交给接口
|
||||
return result
|
||||
|
||||
|
||||
def by_id(result):
|
||||
return {m["metric_id"]: m for m in result["metrics"]}
|
||||
|
||||
|
||||
def exact(value):
|
||||
return Decimal(repr(value)) if isinstance(value, float) else Decimal(value)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 分类：只数据变、只参数变、SQL 结构变
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_same_sql_new_rows_is_data_only():
|
||||
result = run_compare(week("r1"), week("r2", NEXT_ROWS))
|
||||
metrics = by_id(result)
|
||||
east, total = metrics["east"], metrics["total"]
|
||||
assert (east["classification"], total["classification"]) == ("data_only", "data_only")
|
||||
assert (east["base"], east["target"], east["delta"]) == (1200.5, 1500.5, 300.0)
|
||||
assert (east["base_rendered"], east["target_rendered"], east["delta_rendered"]) == ("1,200.5元", "1,500.5元", "+300.0元")
|
||||
assert east["delta_pct"] == round(300 / 1200.5 * 100, 2)
|
||||
assert east["card_id"] == "card" and east["card_title"] == "周报口径卡" and east["name"] == "东区金额"
|
||||
assert east["decomposition"] is None and east["decomposition_note"] is None and east["reasons"]
|
||||
assert total["delta"] == 200.0 and total["delta_rendered"] == "+200.0元"
|
||||
assert result["summary"] == {"unchanged": 0, "data_only": 2, "caliber_and_data": 0, "sql_changed": 0,
|
||||
"params_only": 0, "added": 0, "removed": 0, "incomparable": 0}
|
||||
# 输入两边的值都列出来，带着各自的查询快照
|
||||
[cell] = east["inputs"]
|
||||
assert (cell["path"], cell["base"], cell["target"]) == (EAST, 1200.5, 1500.5)
|
||||
assert cell["base_artifact"] != cell["target_artifact"]
|
||||
# 查询：节点 + 第几次 + 工具配上对，SQL 一字不差
|
||||
assert [(q["node_id"], q["ordinal"], q["structure"]) for q in result["queries"]] == [
|
||||
("fetch", 1, "same"), ("bot", 1, "same")]
|
||||
fetch = result["queries"][0]
|
||||
assert fetch["node_title"] == "区域取数" and fetch["base_sql"] == fetch["target_sql"] == region_sql()
|
||||
assert fetch["literals"] == [] and (fetch["base_rows"], fetch["target_rows"]) == (3, 3)
|
||||
assert (fetch["base_truncated"], fetch["target_truncated"]) == (False, False)
|
||||
assert result["template"]["same_graph"] is True and result["template"]["nodes"] == []
|
||||
assert (result["base"]["id"], result["target"]["id"]) == ("r1", "r2")
|
||||
|
||||
|
||||
def test_identical_runs_are_unchanged():
|
||||
result = run_compare(week("r1"), week("r2"))
|
||||
assert {m["classification"] for m in result["metrics"]} == {"unchanged"}
|
||||
east = by_id(result)["east"]
|
||||
assert east["delta"] == 0 and east["delta_rendered"] is None and east["attribution"] is None
|
||||
assert result["summary"]["unchanged"] == 2
|
||||
|
||||
|
||||
def test_only_date_literals_changed_is_params_only():
|
||||
# 图里的 SQL 是模板（两次一样），执行时代进去的日期不同
|
||||
graph = weekly_graph(region_sql(("{{ input.start }}", "{{ input.end }}")))
|
||||
result = run_compare(week("r1", graph=graph), week("r2", NEXT_ROWS, sql=region_sql(WEEK37), graph=graph))
|
||||
metrics = by_id(result)
|
||||
assert {m["classification"] for m in metrics.values()} == {"params_only"}
|
||||
assert any("参数" in r for r in metrics["east"]["reasons"])
|
||||
fetch = result["queries"][0]
|
||||
assert fetch["structure"] == "params_only"
|
||||
assert fetch["literals"] == [{"base": "'2026-08-31'", "target": "'2026-09-07'"},
|
||||
{"base": "'2026-09-06'", "target": "'2026-09-13'"}]
|
||||
# 定时运行换了日期属于参数变化：模板（图）一样
|
||||
assert result["summary"]["params_only"] == 2 and result["template"]["same_graph"] is True
|
||||
|
||||
|
||||
def test_a_changed_sql_structure_is_flagged_not_split():
|
||||
paid = region_sql(extra=" AND status = 'paid'")
|
||||
result = run_compare(week("r1"), week("r2", NEXT_ROWS, sql=paid))
|
||||
metrics = by_id(result)
|
||||
assert {m["classification"] for m in metrics.values()} == {"sql_changed"}
|
||||
east = metrics["east"]
|
||||
assert east["decomposition"] is None and east["attribution"] is None
|
||||
assert any("区域取数" in r and "SQL" in r for r in east["reasons"])
|
||||
fetch = result["queries"][0]
|
||||
assert fetch["structure"] == "changed" and fetch["literals"] == []
|
||||
assert any("status = 'paid'" in d for d in fetch["differences"])
|
||||
# 图里 fetch 的 SQL 也改了：模板对比列出它和改了的配置项
|
||||
[changed] = result["template"]["nodes"]
|
||||
assert (changed["node_id"], changed["change"], changed["keys"]) == ("fetch", "config", ["args.sql"])
|
||||
assert result["template"]["same_graph"] is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 口径变了：严格拆分
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def net_metrics(expression):
|
||||
"""净额指标 net：表达式由调用方定（v1 只取金额，v2 减去退款）。"""
|
||||
def build(fetch, bot, rows, bot_rows):
|
||||
gmv, refund = rows[0][1], rows[0][2]
|
||||
inputs = [cell_in(0, "gmv", gmv, fetch)]
|
||||
value = gmv
|
||||
if REFUND in expression:
|
||||
inputs.append(cell_in(0, "refund", refund, fetch))
|
||||
value = gmv - refund
|
||||
return [metric("net", expression, value, inputs, name="东区净额")]
|
||||
return build
|
||||
|
||||
|
||||
def test_a_caliber_change_is_split_into_caliber_and_data_effects():
|
||||
base = week("r1", metrics=net_metrics(EAST), version="v1")
|
||||
target = week("r2", NEXT_ROWS, metrics=net_metrics(f"{EAST} - {REFUND}"), version="v2")
|
||||
net = by_id(run_compare(base, target))["net"]
|
||||
assert net["classification"] == "caliber_and_data"
|
||||
assert any("表达式" in r for r in net["reasons"]) and any("v1" in r and "v2" in r for r in net["reasons"])
|
||||
assert (net["base"], net["target"], net["delta"]) == (1200.5, 1380.5, 180.0)
|
||||
split = net["decomposition"]
|
||||
assert net["decomposition_note"] is None
|
||||
# 口径效应 = f_new(x_new) − f_old(x_new) = 1380.5 − 1500.5；数据效应 = f_old(x_new) − f_old(x_old) = 1500.5 − 1200.5
|
||||
assert (split["caliber_effect"], split["data_effect"]) == (-120.0, 300.0)
|
||||
assert (split["caliber_rendered"], split["data_rendered"]) == ("-120.0元", "+300.0元")
|
||||
assert exact(split["caliber_effect"]) + exact(split["data_effect"]) == exact(net["delta"])
|
||||
|
||||
|
||||
def test_the_split_adds_up_exactly_with_awkward_decimals():
|
||||
"""小数位不同、浮点相减有误差的数：两效应之和逐位等于总变化。"""
|
||||
def build(expression, decimals):
|
||||
def make(fetch, bot, rows, bot_rows):
|
||||
gmv, refund = rows[0][1], rows[0][2]
|
||||
inputs = [cell_in(0, "gmv", gmv, fetch), cell_in(0, "refund", refund, fetch)]
|
||||
value = round(eval(expression.replace(EAST, repr(gmv)).replace(REFUND, repr(refund))), decimals)
|
||||
return [metric("ratio", expression, value, inputs, decimals=decimals, unit="")]
|
||||
return make
|
||||
|
||||
base_rows = [["东区", 1000.1, 0.3], *BASE_ROWS[1:]]
|
||||
next_rows = [["东区", 1000.7, 0.2], *NEXT_ROWS[1:]]
|
||||
base = week("r1", base_rows, metrics=build(f"{EAST} * 0.1 - {REFUND}", 2))
|
||||
target = week("r2", next_rows, metrics=build(f"{EAST} * 0.3 + {REFUND}", 3), version="v2")
|
||||
ratio = by_id(run_compare(base, target))["ratio"]
|
||||
split = ratio["decomposition"]
|
||||
assert split is not None, ratio["decomposition_note"]
|
||||
# f_old(x_new) 按旧口径的小数位取整：round(1000.7 * 0.1 - 0.2, 2) = 99.87
|
||||
assert exact(split["data_effect"]) == exact(99.87) - exact(ratio["base"])
|
||||
assert exact(split["caliber_effect"]) + exact(split["data_effect"]) == exact(ratio["delta"])
|
||||
assert exact(ratio["delta"]) == exact(ratio["target"]) - exact(ratio["base"])
|
||||
|
||||
|
||||
def test_a_version_bump_with_the_same_formula_is_all_data():
|
||||
base = week("r1", metrics=net_metrics(EAST), version="v1")
|
||||
target = week("r2", NEXT_ROWS, metrics=net_metrics(EAST), version="v2")
|
||||
net = by_id(run_compare(base, target))["net"]
|
||||
assert net["classification"] == "caliber_and_data"
|
||||
assert (net["decomposition"]["caliber_effect"], net["decomposition"]["data_effect"]) == (0, 300.0)
|
||||
assert net["decomposition"]["caliber_rendered"] == "0.0元"
|
||||
|
||||
|
||||
def test_no_split_when_an_old_leaf_is_not_recorded_this_time():
|
||||
base = week("r1", metrics=net_metrics(f"{EAST} - {REFUND}"), version="v1")
|
||||
target = week("r2", NEXT_ROWS, metrics=net_metrics(EAST), version="v2")
|
||||
net = by_id(run_compare(base, target))["net"]
|
||||
assert net["classification"] == "caliber_and_data" and net["decomposition"] is None
|
||||
assert REFUND in net["decomposition_note"] and "无法严格拆分" in net["decomposition_note"]
|
||||
# 总变化照算
|
||||
assert net["delta"] == 1500.5 - (1200.5 - 100.0)
|
||||
|
||||
|
||||
def test_no_split_when_the_sql_changed_too():
|
||||
base = week("r1", metrics=net_metrics(EAST), version="v1")
|
||||
target = week("r2", NEXT_ROWS, sql=region_sql(extra=" AND status = 'paid'"), metrics=net_metrics(EAST), version="v2")
|
||||
net = by_id(run_compare(base, target))["net"]
|
||||
assert net["classification"] == "caliber_and_data" and net["decomposition"] is None
|
||||
assert "SQL" in net["decomposition_note"]
|
||||
|
||||
|
||||
def test_a_unit_change_is_not_subtracted():
|
||||
def build(unit):
|
||||
def make(fetch, bot, rows, bot_rows):
|
||||
return [metric("east", EAST, rows[0][1], [cell_in(0, "gmv", rows[0][1], fetch)], unit=unit)]
|
||||
return make
|
||||
east = by_id(run_compare(week("r1", metrics=build("元")), week("r2", NEXT_ROWS, metrics=build("万元"))))["east"]
|
||||
assert east["classification"] == "caliber_and_data" and any("单位" in r for r in east["reasons"])
|
||||
assert east["delta"] is None and east["delta_rendered"] is None and east["decomposition"] is None
|
||||
assert "单位" in east["decomposition_note"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 维度归因
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_attribution_pairs_keys_and_shares_add_up_to_one():
|
||||
total = by_id(run_compare(week("r1"), week("r2", NEXT_ROWS)))["total"]
|
||||
attribution = total["attribution"]
|
||||
assert attribution["columns"] == ["region"] and attribution["measure"] == "gmv"
|
||||
assert attribution["total_delta"] == 200.0
|
||||
rows = attribution["rows"]
|
||||
assert [(r["key"], r["base"], r["target"], r["delta"]) for r in rows] == [
|
||||
(["东区"], 1200.5, 1500.5, 300.0), (["南区"], 800.0, 700.0, -100.0), (["西区"], 600.0, 600.0, 0.0)]
|
||||
assert [r["share"] for r in rows] == [1.5, -0.5, 0.0]
|
||||
assert abs(sum(r["share"] for r in rows) - 1) < 1e-12
|
||||
|
||||
|
||||
def test_attribution_pairs_by_key_not_by_row_position():
|
||||
"""这次多了一个区域、顺序也变了：按区域配对，只有一边有的那一边写空。"""
|
||||
next_rows = [["北区", 90.0, 0.0], ["东区", 1500.5, 120.0], ["西区", 600.0, 10.0], ["南区", 700.0, 40.0]]
|
||||
total = by_id(run_compare(week("r1"), week("r2", next_rows)))["total"]
|
||||
rows = {tuple(r["key"]): r for r in total["attribution"]["rows"]}
|
||||
assert rows[("北区",)]["base"] is None and rows[("北区",)]["target"] == 90.0 and rows[("北区",)]["delta"] == 90.0
|
||||
assert rows[("东区",)]["delta"] == 300.0 and rows[("南区",)]["delta"] == -100.0
|
||||
assert abs(sum(r["share"] for r in rows.values()) - 1) < 1e-12
|
||||
|
||||
|
||||
def test_attribution_keeps_the_ten_largest_changes():
|
||||
names = [f"{n}区" for n in "一二三四五六七八九十甲乙"]
|
||||
base_rows = [[name, 100.0, 0.0] for name in names]
|
||||
next_rows = [[name, 100.0 + (i + 1) * 10 * (-1) ** i, 0.0] for i, name in enumerate(names)]
|
||||
total = by_id(run_compare(week("r1", base_rows), week("r2", next_rows)))["total"]
|
||||
rows = total["attribution"]["rows"]
|
||||
assert len(rows) == 10
|
||||
assert [abs(r["delta"]) for r in rows] == [120.0, 110.0, 100.0, 90.0, 80.0, 70.0, 60.0, 50.0, 40.0, 30.0]
|
||||
assert "12" in total["attribution"]["note"]
|
||||
# 占比按全部维度值的总变化算：前 10 个之和不必为 1，全部之和为 1
|
||||
assert total["attribution"]["total_delta"] == sum((i + 1) * 10 * (-1) ** i for i in range(12))
|
||||
|
||||
|
||||
def test_no_share_when_the_measure_does_not_add_up_to_the_input():
|
||||
"""东区金额只是快照里的一格：各区域之和不等于它，各键的变化照列，占比为空并说明。"""
|
||||
east = by_id(run_compare(week("r1"), week("r2", NEXT_ROWS)))["east"]
|
||||
attribution = east["attribution"]
|
||||
assert attribution["measure"] == "gmv" and [r["key"] for r in attribution["rows"]] == [["东区"], ["南区"], ["西区"]]
|
||||
assert all(r["share"] is None for r in attribution["rows"]) and attribution["note"]
|
||||
|
||||
|
||||
def test_a_total_row_is_left_out_of_its_own_breakdown():
|
||||
"""合计行和各区域在同一份结果里：指标取合计那一格，按其余各行归因，占比之和为 1。"""
|
||||
base_rows = [*BASE_ROWS, ["合计", 2600.5, 150.0]]
|
||||
next_rows = [*NEXT_ROWS, ["合计", 2800.5, 170.0]]
|
||||
|
||||
def build(fetch, bot, rows, bot_rows):
|
||||
return [metric("all", "cell(nodes.fetch, 3, 'gmv')", rows[3][1], [cell_in(3, "gmv", rows[3][1], fetch)])]
|
||||
total = by_id(run_compare(week("r1", base_rows, metrics=build), week("r2", next_rows, metrics=build)))["all"]
|
||||
rows = total["attribution"]["rows"]
|
||||
assert ["合计"] not in [r["key"] for r in rows] and len(rows) == 3
|
||||
assert abs(sum(r["share"] for r in rows) - 1) < 1e-12 and total["attribution"]["total_delta"] == 200.0
|
||||
|
||||
|
||||
def test_a_truncated_snapshot_is_not_attributed():
|
||||
base, target = week("r1"), Fake("r2")
|
||||
sql = region_sql()
|
||||
fetch = target.tool("fetch", sql, rows=NEXT_ROWS)
|
||||
bot = target.agent("bot", sql, rows=NEXT_ROWS, truncated=True)
|
||||
target.card(default_metrics(fetch, bot, NEXT_ROWS, NEXT_ROWS))
|
||||
total = by_id(run_compare(base, target))["total"]
|
||||
attribution = total["attribution"]
|
||||
assert attribution["rows"] == [] and "截断" in attribution["note"]
|
||||
assert result_queries(run_compare(base, target), "bot")["target_truncated"] is True
|
||||
|
||||
|
||||
def result_queries(result, node_id):
|
||||
return next(q for q in result["queries"] if q["node_id"] == node_id)
|
||||
|
||||
|
||||
def test_a_period_column_that_never_matches_is_not_a_dimension():
|
||||
"""每期都不同的列（这里是日期写成的文本）不拿来配对：按其余的维度列归因。"""
|
||||
columns = ["day", "region", "gmv"]
|
||||
types = {"day": "text", "region": "text", "gmv": "number"}
|
||||
|
||||
def make(run_id, days, values):
|
||||
run = Fake(run_id)
|
||||
rows = [[day, region, value] for day in days for region, value in zip(("东区", "南区"), values)]
|
||||
sql = "SELECT day, region, SUM(amount) AS gmv FROM orders GROUP BY day, region"
|
||||
bot = run.agent("bot", sql, columns, rows, types=types)
|
||||
total = sum(r[2] for r in rows)
|
||||
run.card([metric("total", "sum(vars.kpi.amounts)", total, [amounts_in(bot, rows)])])
|
||||
return run
|
||||
|
||||
total = by_id(run_compare(make("r1", ("2026-09-05", "2026-09-06"), (10.0, 20.0)),
|
||||
make("r2", ("2026-09-12", "2026-09-13"), (15.0, 18.0))))["total"]
|
||||
attribution = total["attribution"]
|
||||
assert attribution["columns"] == ["region"]
|
||||
assert [(r["key"], r["delta"]) for r in attribution["rows"]] == [(["东区"], 10.0), (["南区"], -4.0)]
|
||||
assert "day" in attribution["note"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 遮罩列
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_masked_columns_never_show_up():
|
||||
"""数据源遮了 manager（经理名）和 refund：经理名不当维度、不出现在结果里；用到 refund 的指标输入不给值、不拆分。"""
|
||||
columns = ["region", "manager", "gmv", "refund"]
|
||||
types = {"region": "text", "manager": "text", "gmv": "number", "refund": "number"}
|
||||
|
||||
def make(run_id, rows, expression, version):
|
||||
run = Fake(run_id)
|
||||
sql = "SELECT region, manager, SUM(amount) AS gmv, SUM(refund) AS refund FROM orders GROUP BY region, manager"
|
||||
fetch = run.tool("fetch", sql, columns, rows, types=types)
|
||||
bot = run.agent("bot", sql, columns, rows, types=types)
|
||||
run.masked = {fetch: frozenset({"manager", "refund"}), bot: frozenset({"manager", "refund"})}
|
||||
inputs = [cell_in(0, "gmv", rows[0][2], fetch)]
|
||||
value = rows[0][2]
|
||||
if "refund" in expression:
|
||||
inputs.append(cell_in(0, "refund", rows[0][3], fetch))
|
||||
value = rows[0][2] - rows[0][3]
|
||||
run.card([metric("net", expression, value, inputs),
|
||||
metric("total", "sum(vars.kpi.amounts)", sum(r[2] for r in rows), [amounts_in(bot, rows)])],
|
||||
version=version)
|
||||
return run
|
||||
|
||||
base_rows = [["东区", "经理甲", 1200.5, 101.25], ["南区", "经理乙", 800.0, 52.75]]
|
||||
next_rows = [["东区", "经理甲", 1500.5, 133.5], ["南区", "经理乙", 700.0, 47.25]]
|
||||
result = run_compare(make("r1", base_rows, EAST, "v1"),
|
||||
make("r2", next_rows, "cell(nodes.fetch, 0, 'gmv') - cell(nodes.fetch, 0, 'refund')", "v2"))
|
||||
text = json.dumps(result, ensure_ascii=False)
|
||||
for secret in ("经理甲", "经理乙", "101.25", "52.75", "133.5", "47.25"):
|
||||
assert secret not in text, secret
|
||||
metrics = by_id(result)
|
||||
net = metrics["net"]
|
||||
refund = next(i for i in net["inputs"] if "refund" in i["path"])
|
||||
assert refund["masked"] is True and refund["base"] is None and refund["target"] is None
|
||||
assert net["decomposition"] is None and "遮罩" in net["decomposition_note"]
|
||||
# 维度只用没遮的 region；占比照给
|
||||
attribution = metrics["total"]["attribution"]
|
||||
assert attribution["columns"] == ["region"] and abs(sum(r["share"] for r in attribution["rows"]) - 1) < 1e-12
|
||||
assert any("遮罩" in n for n in result["notes"])
|
||||
|
||||
|
||||
def test_a_masked_measure_is_not_attributed():
|
||||
def masked(run):
|
||||
run.masked = {a: frozenset({"gmv"}) for a in run.queries}
|
||||
return run
|
||||
result = run_compare(masked(week("r1")), masked(week("r2", NEXT_ROWS)))
|
||||
total = by_id(result)["total"]
|
||||
assert total["attribution"]["rows"] == [] and "遮罩" in total["attribution"]["note"]
|
||||
east = by_id(result)["east"]
|
||||
assert east["inputs"][0]["base"] is None and east["inputs"][0]["masked"] is True
|
||||
assert "1200.5" not in json.dumps(east["inputs"]) and "800.0" not in json.dumps(result)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 对不上具体查询的输入、运行输入、一边才有的指标
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def code_metrics(value):
|
||||
def build(fetch, bot, rows, bot_rows):
|
||||
return [metric("calc", "vars.adj * 2", value * 2, [scalar_in("vars.adj", value, "calc", "code")])]
|
||||
return build
|
||||
|
||||
|
||||
def test_an_input_from_a_code_node_follows_its_upstream():
|
||||
def make(run_id, rows, value, sql=None):
|
||||
run = Fake(run_id, graph=weekly_graph(sql, between=(node("calc", "code", "调整", code="print(1)",
|
||||
assign_to="adj"),)))
|
||||
sql = sql or region_sql()
|
||||
run.tool("fetch", sql, rows=rows)
|
||||
run.agent("bot", sql, rows=rows)
|
||||
run.code("calc", value, "adj")
|
||||
run.card(code_metrics(value)(None, None, rows, rows))
|
||||
return run
|
||||
|
||||
# 上游什么都没变：变化来自数据
|
||||
calc = by_id(run_compare(make("r1", BASE_ROWS, 10), make("r2", NEXT_ROWS, 12)))["calc"]
|
||||
assert calc["classification"] == "data_only" and calc["delta"] == 4
|
||||
# 上游的 SQL 结构变了：说不清它变没变，不能报成数据变化
|
||||
calc = by_id(run_compare(make("r1", BASE_ROWS, 10),
|
||||
make("r2", NEXT_ROWS, 12, sql=region_sql(extra=" AND status = 'paid'"))))["calc"]
|
||||
assert calc["classification"] == "incomparable" and any("调整" in r for r in calc["reasons"])
|
||||
|
||||
|
||||
def test_the_code_node_itself_changed_is_a_template_change():
|
||||
def make(run_id, code, value):
|
||||
run = Fake(run_id, graph=weekly_graph(between=(node("calc", "code", "调整", code=code, assign_to="adj"),)))
|
||||
run.tool("fetch", region_sql())
|
||||
run.agent("bot", region_sql())
|
||||
run.code("calc", value, "adj")
|
||||
run.card(code_metrics(value)(None, None, BASE_ROWS, BASE_ROWS))
|
||||
return run
|
||||
calc = by_id(run_compare(make("r1", "x = 1", 10), make("r2", "x = 2", 11)))["calc"]
|
||||
assert calc["classification"] == "sql_changed" and any("调整" in r for r in calc["reasons"])
|
||||
|
||||
|
||||
def test_a_run_input_change_is_a_parameter_change():
|
||||
def make(run_id, goal):
|
||||
run = Fake(run_id, inputs={"goal": goal})
|
||||
run.add("node.started", "start", {"node_type": "input"})
|
||||
run.add("node.finished", "start", {"artifact": run.put({"goal": goal})})
|
||||
fetch = run.tool("fetch", region_sql())
|
||||
run.agent("bot", region_sql())
|
||||
run.card([metric("reach", f"round({EAST} / input.goal * 100, 1)", round(1200.5 / goal * 100, 1),
|
||||
[cell_in(0, "gmv", 1200.5, fetch), scalar_in("input.goal", goal, "start", "input")],
|
||||
unit="%", fmt="plain")])
|
||||
return run
|
||||
result = run_compare(make("r1", 1000), make("r2", 1500))
|
||||
reach = by_id(result)["reach"]
|
||||
assert reach["classification"] == "params_only" and any("input.goal" in r for r in reach["reasons"])
|
||||
assert reach["delta_rendered"] == "-40.0 个百分点"
|
||||
assert any("goal" in n for n in result["notes"])
|
||||
|
||||
|
||||
def test_metrics_and_cards_on_one_side_only():
|
||||
def extra(fetch, bot, rows, bot_rows):
|
||||
return [*default_metrics(fetch, bot, rows, bot_rows),
|
||||
metric("west", "cell(nodes.fetch, 2, 'gmv')", rows[2][1], [cell_in(2, "gmv", rows[2][1], fetch)])]
|
||||
base = week("r1")
|
||||
target = week("r2", metrics=extra)
|
||||
target.card([metric("cnt", "len(vars.kpi.amounts)", 3, [amounts_in(target.queries[1], BASE_ROWS)], unit="个",
|
||||
decimals=None)], node_id="card2", caliber="补充口径")
|
||||
result = run_compare(base, target)
|
||||
metrics = by_id(result)
|
||||
assert metrics["west"]["classification"] == "added" and metrics["west"]["base"] is None
|
||||
assert metrics["cnt"]["classification"] == "added" and metrics["cnt"]["card_id"] == "card2"
|
||||
assert result["summary"]["added"] == 2
|
||||
swapped = by_id(run_compare(target, base))
|
||||
assert swapped["west"]["classification"] == "removed" and swapped["west"]["target"] is None
|
||||
|
||||
|
||||
def test_an_unreadable_card_makes_its_metrics_incomparable():
|
||||
base, target = week("r1"), week("r2", NEXT_ROWS)
|
||||
base.tampered.add(base.cards[0][1])
|
||||
base.cards = [] # 接口只交回复验过哈希的口径卡
|
||||
result = run_compare(base, target)
|
||||
assert {m["classification"] for m in result["metrics"]} == {"incomparable"}
|
||||
assert any("哈希" in r for r in by_id(result)["east"]["reasons"])
|
||||
|
||||
|
||||
def test_a_card_executed_several_times_uses_the_last_one():
|
||||
base = week("r1")
|
||||
target = Fake("r2")
|
||||
fetch = target.tool("fetch", region_sql(), rows=NEXT_ROWS)
|
||||
bot = target.agent("bot", region_sql(), rows=NEXT_ROWS)
|
||||
target.card(default_metrics(fetch, bot, BASE_ROWS, BASE_ROWS))
|
||||
target.card(default_metrics(fetch, bot, NEXT_ROWS, NEXT_ROWS), runs=2)
|
||||
result = run_compare(base, target)
|
||||
assert by_id(result)["east"]["target"] == 1500.5
|
||||
assert any("执行了 3 次" in n for n in result["notes"])
|
||||
|
||||
|
||||
def test_verify_queries_are_left_out():
|
||||
base = week("r1")
|
||||
target = Fake("r2")
|
||||
fetch = target.tool("fetch", region_sql(), rows=NEXT_ROWS)
|
||||
bot = target.agent("bot", region_sql(), rows=NEXT_ROWS, verify=["SELECT COUNT(*) AS n FROM orders"])
|
||||
target.card(default_metrics(fetch, bot, NEXT_ROWS, NEXT_ROWS))
|
||||
result = run_compare(base, target)
|
||||
assert [(q["node_id"], q["structure"]) for q in result["queries"]] == [("fetch", "same"), ("bot", "same")]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 查询配对、模板对比
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_queries_fall_back_to_normalized_sql_pairing():
|
||||
"""节点换了 ID：按「节点 + 第几次 + 工具」配不上，再按规范化 SQL 配（大小写、空白不同也认）。"""
|
||||
base = week("r1")
|
||||
target = Fake("r2", graph=weekly_graph())
|
||||
target.tool("pull", region_sql(WEEK37).lower(), rows=NEXT_ROWS)
|
||||
target.agent("bot", region_sql(WEEK37), rows=NEXT_ROWS)
|
||||
result = run_compare(base, target)
|
||||
pull = result_queries(result, "pull")
|
||||
assert pull["structure"] == "params_only" and pull["base_sql"] == region_sql()
|
||||
assert len(result["queries"]) == 2
|
||||
|
||||
|
||||
def test_unpaired_queries_are_added_or_removed():
|
||||
base = week("r1")
|
||||
target = week("r2")
|
||||
target.tool("extra", "SELECT COUNT(*) AS n FROM refunds", ["n"], [[7]], types={"n": "number"})
|
||||
result = run_compare(base, target)
|
||||
extra = result_queries(result, "extra")
|
||||
assert extra["structure"] == "added" and extra["base_sql"] is None and extra["target_rows"] == 1
|
||||
removed = result_queries(run_compare(target, base), "extra")
|
||||
assert removed["structure"] == "removed" and removed["target_sql"] is None
|
||||
|
||||
|
||||
def test_the_template_lists_added_removed_and_reconfigured_nodes():
|
||||
base_graph = weekly_graph(between=(node("calc", "code", "调整", code="x = 1"),))
|
||||
target_graph = weekly_graph(region_sql(WEEK37), between=(node("check", "dual_check", "双盲复算"),))
|
||||
target_graph["nodes"][-2]["data"]["label"] = "写周报（新）"
|
||||
target_graph["nodes"][-1]["data"]["config"] = {"fields": [{"name": "周报", "value": "{{ nodes.write.text }}"}]}
|
||||
base = week("r1", graph=base_graph)
|
||||
target = week("r2", graph=target_graph, version="v1")
|
||||
base.version, target.version = 3, 4
|
||||
template = run_compare(base, target)["template"]
|
||||
assert (template["base_version"], template["target_version"], template["same_graph"]) == (3, 4, False)
|
||||
changes = {n["node_id"]: (n["change"], n["keys"]) for n in template["nodes"]}
|
||||
assert changes == {"fetch": ("config", ["args.sql"]), "check": ("added", []), "write": ("config", ["label"]),
|
||||
"out": ("config", ["fields"]), "calc": ("removed", [])}
|
||||
labels = {n["node_id"]: n["labels"] for n in template["nodes"]}
|
||||
assert labels["write"] == ["节点名称"] and labels["out"] == ["成果字段"]
|
||||
# 配置的值（这里是 SQL 和模板）一个字都不展示
|
||||
assert "2026-09" not in json.dumps(template, ensure_ascii=False)
|
||||
|
||||
|
||||
def test_a_missing_snapshot_file_is_reported_and_withheld():
|
||||
"""基准运行的快照文件不在了：SQL 从 tool.start 的参数里取，结构照比；输入的值核对不了遮没遮，不给；不归因。"""
|
||||
base, target = week("r1"), week("r2", NEXT_ROWS)
|
||||
fetch = base.queries[0]
|
||||
del base.store[fetch]
|
||||
result = run_compare(base, target)
|
||||
query = result_queries(result, "fetch")
|
||||
assert query["structure"] == "same" and query["base_sql"] == region_sql() and query["base_rows"] is None
|
||||
assert "无法读取" in query["problem"]
|
||||
east = by_id(result)["east"]
|
||||
assert east["classification"] == "data_only" and east["inputs"][0]["base"] is None
|
||||
assert east["attribution"]["rows"] == [] and "无法读取" in east["attribution"]["note"]
|
||||
|
||||
|
||||
def test_runs_without_metric_cards_still_compare_queries():
|
||||
base, target = Fake("r1"), Fake("r2")
|
||||
base.tool("fetch", region_sql())
|
||||
target.tool("fetch", region_sql(WEEK37), rows=NEXT_ROWS)
|
||||
result = run_compare(base, target)
|
||||
assert result["metrics"] == [] and sum(result["summary"].values()) == 0
|
||||
assert any("没有口径卡" in n for n in result["notes"])
|
||||
assert [q["structure"] for q in result["queries"]] == ["params_only"]
|
||||
|
||||
|
||||
def test_the_comparison_does_not_touch_the_runs():
|
||||
base, target = week("r1"), week("r2", NEXT_ROWS)
|
||||
before = (list(base.events), list(target.events), dict(base.store), dict(target.store))
|
||||
run_compare(base, target)
|
||||
assert (base.events, target.events, base.store, target.store) == before
|
||||
Reference in New Issue
Block a user