mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 01:51:57 +00:00
合入 nr6/a：认口径按答案引用的那几列投影后再判等价（判定仍只走现有的 match_template）、差异说明投影之后再说并认得 LEFT JOIN ON 里的日期条件、转抄口径卡指标的出处跨卡追到取数查询（对账、召回不再降档）、认证队列按规范形分组、问题聚类的类 id 由成员定、use_caliber 的协议写清 card
This commit is contained in:
ThearchyHelios
@@ -71,13 +71,17 @@ LOOK_WHEN_ANSWER_PROMPT = LOOK_WHEN_PROMPT.replace(
|
||||
口径，交给 agent 去查，系统会自动拿它的 SQL 和口径对账。查库交给 agent 时它运行时会自己查表结构：只有从概览看不出
|
||||
该用哪个库、用户点了表名时才看表，把看到的表名写进 agent 的提示词""")
|
||||
|
||||
#: 问数据里输出协议多的一段：选用口径（ledger/compose.py）。只在这一轮能看已发布口径时接在协议后面
|
||||
#: 问数据里输出协议多的一段：选用口径（ledger/compose.py）。只在这一轮能看已发布口径时接在协议后面。
|
||||
#: 两个 id 写清楚（真实验收里模型第一次常把 card 写成自己起的 c1，被拒之后才改对）：card 是那一版工作流图里口径卡的节点 id，
|
||||
#: 照抄查看结果；id 是这次选用自己起的名字，生成的口径卡用它当节点 id
|
||||
USE_CALIBER_PROMPT = """- 选用已发布口径（只在问数据里、look 过 caliber 之后）：
|
||||
{"op":"use_caliber","id":"c1","workflow_id":"…","version":3,"card":"口径卡节点 id","metrics":["指标id"],"inputs":{"字段":"值"}}
|
||||
照抄查看结果里给的写法，只改 id、metrics、inputs。系统按那一版生成取数节点（c1_<原 id>）和口径卡（节点 id 就是 c1），
|
||||
并把入口连到取数、取数连到口径卡；你**不要**自己写这几个节点，也不要改它们。接下来只需把口径卡连到报告
|
||||
（add_edge c1 → report），报告里用 [[m:指标id]] 引用。inputs 按口径的入口字段填（日期、门店这类问题里给出的条件），
|
||||
填不对会被拒收、原因会交回给你改；口径对不上问法时不要选用"""
|
||||
{"op":"use_caliber","id":"c1","workflow_id":"…","version":3,"card":"card","metrics":["指标id"],"inputs":{"字段":"值"}}
|
||||
照抄查看结果里给的那一行写法：workflow_id、version、card 原样照抄，只改 id、metrics、inputs。两个 id 别弄混：
|
||||
· card 填那一版工作流图里口径卡的节点 id（查看结果的写法里已经写好，例如 "card"），指明选哪一张卡，不是你起的名字；
|
||||
· id 是你给这次选用起的名字（简短英文小写，例如 c1），系统生成的口径卡就用它当节点 id，取数节点是 c1_<原 id>。
|
||||
系统把入口连到取数、取数连到口径卡；你**不要**自己写这几个节点，也不要改它们。接下来只需把口径卡连到报告
|
||||
（add_edge c1 → report，这里的 c1 是你起的 id），报告里用 [[m:指标id]] 引用。inputs 按口径的入口字段填（日期、门店这类
|
||||
问题里给出的条件），填不对会被拒收、原因会交回给你改；口径对不上问法时不要选用"""
|
||||
|
||||
#: 系统代为查看时写进记录的需求（界面上显示）
|
||||
SYSTEM_LOOK_FIX_NEED = "要修的工作流里，查库节点的 SQL 用到的表"
|
||||
|
||||
@@ -789,8 +789,7 @@ async def evidence_segment(run_id: str, segment_id: str, report: str | None = No
|
||||
chain = await _chain(seg, doc, sealed, chosen.node_id)
|
||||
covered = sealed.trusted and all(step.get("sealed") for step in chain if "sealed" in step)
|
||||
# 遮罩列：查询步骤里遮着的，加上指标步骤里取自遮罩列的输入（按位置取的那一格可能没有对应的查询步骤）
|
||||
masked = list(dict.fromkeys(c for step in chain if step.get("step") in ("query", "input")
|
||||
for c in step.get("masked") or []))
|
||||
masked = _chain_masked(chain)
|
||||
verdict = _effective_verdicts(chosen, sealed)[0].get(str(unit.get("id")))
|
||||
challenged = _challenge_of(sealed, chosen, str(unit.get("id")))
|
||||
# 口径账本（next-round 02 做法 3）：这个数来自哪一份口径。比对在封存之后做、不在封存范围内，标签里写明；没比对过的运行
|
||||
@@ -823,6 +822,14 @@ async def evidence_segment(run_id: str, segment_id: str, report: str | None = No
|
||||
METRIC_NOT_FOUND = "evidence_metric_not_found"
|
||||
|
||||
|
||||
def _chain_masked(chain: list[dict[str, Any]]) -> list[str]:
|
||||
"""出处链里遮着的列：查询步骤里遮着的，指标步骤里取自遮罩列的输入（按位置取的那一格可能没有对应的查询步骤），转抄的
|
||||
上游指标那几步的输入（跨卡往下追的，见 _upstream_steps）。"""
|
||||
steps = [*chain, *(i for step in chain if step.get("step") == "upstream" for i in step.get("inputs") or [])]
|
||||
return list(dict.fromkeys(c for step in steps if step.get("step") in ("query", "input")
|
||||
for c in step.get("masked") or []))
|
||||
|
||||
|
||||
@router.get("/{run_id}/evidence/metrics/{node}/{metric}")
|
||||
async def evidence_metric(run_id: str, node: str, metric: str) -> dict[str, Any]:
|
||||
"""点开应用页上的一个指标（应用平台 §3.6）：指标 → 输入 → 查询快照的出处链，和报告片段的指标链同一套（_metric_chain）。
|
||||
@@ -847,8 +854,7 @@ async def evidence_metric(run_id: str, node: str, metric: str) -> dict[str, Any]
|
||||
chain = await _metric_chain({"text": shown}, {"kind": "metric", "locator": {"metric": metric}, "eid": eid},
|
||||
entry, {"catalog": {}}, sealed, masks)
|
||||
covered = sealed.trusted and all(step.get("sealed") for step in chain if "sealed" in step)
|
||||
masked = list(dict.fromkeys(c for step in chain if step.get("step") in ("query", "input")
|
||||
for c in step.get("masked") or []))
|
||||
masked = _chain_masked(chain)
|
||||
return {"run_id": run_id, "node": node, "metric": metric, "rendered": shown, "eid": eid, "chain": chain,
|
||||
"seal": {"sealed": sealed.sealed, "ok": sealed.seal["ok"], "covered": covered},
|
||||
"redacted": {"columns": masked, "note": MASK_NOTE} if masked else {"columns": []}}
|
||||
@@ -992,6 +998,8 @@ async def _metric_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[s
|
||||
if (found := held.get(str(item.get("path")))) is not None:
|
||||
one = _withheld(one, found)
|
||||
inputs.append(one)
|
||||
# 转抄另一张口径卡的指标（对账卡、召回卡的「口径」那一项）：跨卡往下追，接上那个指标自己的一段出处，一直到查询格子
|
||||
upstream = await _upstream_steps(metric or {}, sealed, masks, catalog, wanted, {(artifact, str(metric_id))})
|
||||
if withheld := {path: _held_word(found) for path, found in held.items() if found.kind != "table"}:
|
||||
# 代入式里那几处同样写「已遮罩」（遮没遮核对不了的写「无法核对」）：不然标签里遮着、代入式里亮着，同一个值换个
|
||||
# 地方照样看得到。整份结果当输入的不换：代入式里本来就只留着它的路径
|
||||
@@ -1002,7 +1010,77 @@ async def _metric_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[s
|
||||
# 目录、正文里本来就是「已遮罩」。文档记着不给、这里却认不出的（数据源事后撤了遮罩），照文档不给
|
||||
step["value"] = word or MASKED
|
||||
queries = [s for a, cells in wanted.items() for s in await _query_steps(a, cells, doc, sealed, masks)]
|
||||
return [step, *inputs, *queries]
|
||||
return [step, *inputs, *upstream, *queries]
|
||||
|
||||
|
||||
async def _upstream_steps(metric: dict[str, Any], sealed: _Sealed, masks: "_Masks", catalog: dict[str, Any],
|
||||
wanted: dict[str, list[tuple[Any, Any]]], seen: set[tuple[str, str]],
|
||||
depth: int = 1) -> list[dict[str, Any]]:
|
||||
"""指标的输入里转抄另一张口径卡的指标的（nodes.<卡>.metrics[i].value，卡上记着 upstream），每个接一步 upstream：那张
|
||||
卡的那个指标（名字、值、式子、代入式、核对状态，取值按同一套遮罩），它自己的输入（同指标步骤的输入；落在查询快照上的
|
||||
并进 wanted，后面照常给查询步骤），追没追到原始数据。转抄的转抄一层层接着追（防环：同一个指标只接一次；最多
|
||||
MAX_COPY_DEPTH 层）。of 是这一步接在哪个输入后面（输入的路径）。"""
|
||||
from app.engine.lineage import MAX_COPY_DEPTH, SNAPSHOT_VIAS
|
||||
|
||||
out: list[dict[str, Any]] = []
|
||||
if depth > MAX_COPY_DEPTH:
|
||||
return out
|
||||
for item in metric.get("inputs") or []:
|
||||
up = item.get("upstream") if isinstance(item, dict) and item.get("via") == "metric" else None
|
||||
if not isinstance(up, dict) or not up.get("artifact"):
|
||||
continue
|
||||
artifact, metric_id = str(up["artifact"]), str(up.get("metric") or "")
|
||||
if (artifact, metric_id) in seen:
|
||||
continue
|
||||
seen.add((artifact, metric_id))
|
||||
node = str(item.get("node_id") or "")
|
||||
card, hash_ok = _load_card(artifact)
|
||||
found = _metric_in(card, metric_id)
|
||||
step: dict[str, Any] = {
|
||||
"step": "upstream", "of": item.get("path"), "node_id": node or None,
|
||||
"label": _node_label(sealed, node) if node else None, "artifact": artifact, "metric": metric_id,
|
||||
"traced": up.get("traced") is True, "depth": depth, "hash_ok": hash_ok and found is not None,
|
||||
"sealed": sealed.trusted and (node, artifact) in sealed.ledger,
|
||||
}
|
||||
if found is None:
|
||||
out.append(step)
|
||||
continue
|
||||
held, word = await _metric_masks(found, node, artifact, sealed, masks, card=card)
|
||||
try:
|
||||
rendered = word or render_metric(found)
|
||||
except RenderError:
|
||||
rendered = MISSING
|
||||
step.update({
|
||||
"name": found.get("name") or metric_id, "value": word if word is not None else found.get("value"),
|
||||
"unit": found.get("unit") or "", "rendered": rendered, "caliber": (card or {}).get("caliber"),
|
||||
"version": (card or {}).get("caliber_version"), "source": await _caliber_source((card or {}).get("source")),
|
||||
"expression": found.get("expression"), "substituted": found.get("substituted"),
|
||||
"recompute_ok": found.get("recompute_ok"), "status": found.get("status"),
|
||||
**({"incomplete": True, "incomplete_reason": found.get("incomplete_reason") or ""}
|
||||
if found.get("incomplete") else {}),
|
||||
**({"sql_check_failed": True, "sql_check_reason": found.get("sql_check_reason") or ""}
|
||||
if found.get("sql_check_failed") else {}),
|
||||
"untraced": _untraced_steps(found, sealed),
|
||||
})
|
||||
if withheld := {path: _held_word(h) for path, h in held.items() if h.kind != "table"}:
|
||||
step["substituted"] = recompute.substituted_without(found, withheld)
|
||||
inputs: list[dict[str, Any]] = []
|
||||
for sub in found.get("inputs") or []:
|
||||
if not isinstance(sub, dict):
|
||||
continue
|
||||
one = {"step": "input", **sub}
|
||||
if sub.get("artifact") and sub.get("via") in SNAPSHOT_VIAS:
|
||||
one["query"] = _query_alias(catalog, sub["artifact"])
|
||||
if cell := _input_cell(sub, catalog):
|
||||
one["cell"] = cell
|
||||
wanted.setdefault(str(sub["artifact"]), []).extend(_located(sub.get("locator") or {}))
|
||||
if (h := held.get(str(sub.get("path")))) is not None:
|
||||
one = _withheld(one, h)
|
||||
inputs.append(one)
|
||||
step["inputs"] = inputs
|
||||
out.append(step)
|
||||
out += await _upstream_steps(found, sealed, masks, catalog, wanted, seen, depth + 1)
|
||||
return out
|
||||
|
||||
|
||||
def _node_label(sealed: _Sealed, node_id: Any) -> str:
|
||||
|
||||
@@ -249,7 +249,10 @@ async def certify_dismiss(payload: DismissIn, session: AsyncSession = Depends(ge
|
||||
"""「不是口径」：记一条审计（ledger.certify.dismiss），这一项之后不再进队。"""
|
||||
from app.ledger import certify
|
||||
|
||||
from app.api.deps import current_principal
|
||||
|
||||
if payload.kind is not None and payload.kind not in certify.KINDS:
|
||||
raise HTTPException(422, "可沉淀的问题只有三种：确定性重复、问题聚类、被反复用的答案")
|
||||
await certify.dismiss(session, payload.item_id, kind=payload.kind, note=payload.note)
|
||||
await certify.dismiss(session, payload.item_id, kind=payload.kind, note=payload.note,
|
||||
principal=current_principal())
|
||||
return {"ok": True, "item_id": payload.item_id}
|
||||
|
||||
@@ -18,6 +18,12 @@
|
||||
聚类用 memory/embeddings 的向量贪心聚：按时间从新到旧，和某一类的种子足够像就并进去，否则自立一类。本地向量是词频
|
||||
哈希、语义能力弱，结果是下限——调用方据 embedder 照实写明用的是哪个向量模型。
|
||||
|
||||
**类的 id 由成员定。** 每个问题有稳定的 id（Question.key：问数据是那一轮，别的探索运行是那次运行），一类的 id
|
||||
（QuestionCluster.id）是成员的 id 排好之后取的哈希：同一批问题两次读取，类的 id 一样。以前按种子问题的原文算，种子是一类里
|
||||
最新的那一句，同一批问题时间相同的几句先后不定、来了一句新的问法种子就换了，「不是口径」按 id 记就失效了。同一时刻问的
|
||||
几句按 id 排出先后（recent_questions），不靠库里读出来的先后。来了新成员 id 照样会变：认证队列按成员记「不是口径」
|
||||
（ledger/certify），不只按 id。
|
||||
|
||||
**向量只取一次。** 同一批问题会被聚很多遍：评测页「口径命中与重复度」一次打开，/api/evals/signals 按窗口调 compute_stats
|
||||
（缺省 6 个窗口，每个都聚一次），页面另调 /api/ledger/stats；认证队列、治理页的探索问题聚类也各聚一遍。配了远程向量模型时
|
||||
每聚一遍都把几百句问题重新发一次，页面慢、还花钱。所以问题原文 → 向量在进程里留一份（_vectors：最多 VECTOR_CACHE_MAX 句，
|
||||
@@ -28,6 +34,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import weakref
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Sequence
|
||||
@@ -55,6 +62,16 @@ class Question:
|
||||
workflow: str | None = None
|
||||
origin: str | None = None
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
"""稳定的 id：问数据的那一轮（turn:…），别的探索运行是那次运行（run:…）；两样都没有的按原文和时间。"""
|
||||
if self.turn_id:
|
||||
return f"turn:{self.turn_id}"
|
||||
if self.run_id:
|
||||
return f"run:{self.run_id}"
|
||||
stamp = self.at.isoformat() if self.at is not None else ""
|
||||
return "text:" + hashlib.sha256(f"{self.text}\n{stamp}".encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
@dataclass
|
||||
class QuestionCluster:
|
||||
@@ -66,6 +83,16 @@ class QuestionCluster:
|
||||
def count(self) -> int:
|
||||
return len(self.members)
|
||||
|
||||
@property
|
||||
def keys(self) -> list[str]:
|
||||
"""成员的稳定 id，排好的。"""
|
||||
return sorted(q.key for q in self.members)
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
"""类的 id：成员的稳定 id 排好之后取哈希（见模块说明）。同一批问题两次读取一样。"""
|
||||
return hashlib.sha256("\n".join(self.keys).encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
async def recent_questions(session: AsyncSession, principal: Any, *, limit: int) -> list[Question]:
|
||||
"""最近问过的问题，从新到旧，最多 limit 条（取法见模块开头）。principal 用来去掉这个人读不了的只给拥有者的运行。"""
|
||||
@@ -94,7 +121,8 @@ async def recent_questions(session: AsyncSession, principal: Any, *, limit: int)
|
||||
text = (payload or {}).get("question") if isinstance(payload, dict) else None
|
||||
if isinstance(text, str) and text.strip():
|
||||
out.append(Question(text=text.strip(), at=at, run_id=run_id, workflow=workflow, origin=origin))
|
||||
out.sort(key=lambda q: q.at.timestamp() if q.at is not None else 0.0, reverse=True)
|
||||
# 同一时刻问的几句按稳定的 id 排：库里读出来的先后不定，种子、聚类的结果就跟着变（见模块说明）
|
||||
out.sort(key=lambda q: (q.at.timestamp() if q.at is not None else 0.0, q.key), reverse=True)
|
||||
return out[:limit]
|
||||
|
||||
|
||||
|
||||
@@ -14,6 +14,10 @@ A / B，只差一个方面、又对得上数据目录里的事实（时间归属
|
||||
（规范形式一字不差）才返回空列表。
|
||||
- facets：把一条查询拆成表、连接、WHERE 合取式、时间条件、分组、聚合、算出来的列、HAVING、取哪几行（给调用方取
|
||||
结构用；CTE、子查询按位置编号，见 _Statement）。facets 不受下面等价判定的改动影响。
|
||||
- bind_template / match_template：口径模板的参数绑定与等价证明（口径账本，见那一节）。
|
||||
- project_select / project_template：按引用的列投影——去掉最外层选择列表里没被引用、去掉也不改变被引用那几列取值的
|
||||
选择项，认不准的一律不投影（见那一节）。投影本身不判等价，投影后的 SQL 照样交给上面的判等价。
|
||||
- sql_shape：同一个形状（认证队列的确定性重复）：条件的先后、交换律成立处两边的先后、字面量的值都不算（见那一节）。
|
||||
|
||||
**等价怎么判。** 过去几轮按方面归纳任意 SQL，每轮修好一批又冒出一批（还回退过），收不拢；现在先判等价、再说差在哪。
|
||||
两边各自规范化，规范化的每一步都只做保持语义的变换，同一级的两个规范形式一字不差（比语法树逐个节点的类型和参数，
|
||||
@@ -7037,6 +7041,8 @@ class _Explainer:
|
||||
pb, jb = self._shape(B)
|
||||
if pa == pb and ja == jb:
|
||||
return True, {}, set(), set()
|
||||
if (said := self._left_on_times(A, B, pa, ja, pb, jb)) is not None:
|
||||
return said
|
||||
if not ja and not jb:
|
||||
only_a = [r for r in A.rels if r.alias not in pb]
|
||||
only_b = [r for r in B.rels if r.alias not in pa]
|
||||
@@ -7066,6 +7072,80 @@ class _Explainer:
|
||||
x = flipped[0]
|
||||
return self._left_inner(A, B, x, 0) if kinds_a[x] == "left" else self._left_inner(B, A, x, 1)
|
||||
|
||||
def _left_on_times(self, A: _Block, B: _Block, pa: Any, ja: Any, pb: Any, jb: Any
|
||||
) -> tuple[bool, dict[str, str], set[str], set[str]] | None:
|
||||
"""来源两边一样，只差在一个 LEFT JOIN 的 ON 里的时间条件（日期条件写在 LEFT JOIN 的 ON 里）：说「A 用 LEFT JOIN
|
||||
关联 X 表时只关联 … 的行，B 没有这个条件」「A 用 LEFT JOIN 关联 X 表时对 paid_at 的条件是 …，B 是 …」「A … 时按
|
||||
paid_at 筛时间，B 按 created_at」。说不准的返回 None（照旧整条兜底）。
|
||||
|
||||
说得准要（每一条都是为了不说出不存在的不同）：开头那组来源一样，后面的连接只有这一个的 ON 不同（别名、表、LEFT 都
|
||||
一样）；两边 ON 里没配上的条件每一个都是时间条件（timed）、只涉及这张可空一侧的表的列；这张表在 ON 之外只出现在
|
||||
结果里（SELECT 列表、分组、排序），没在 WHERE、HAVING、别的连接里用到（用到了就是反连接、或者关联条件的事）；
|
||||
SELECT 列表里用到了它的列（不然 ON 里的不同只影响行会不会重复，说不准）；结果看得见行；这张表在这一段只出现一次。
|
||||
两边都有、同一列上的，按取值逐段算过留下的取值确实不同（_Range.compare，同 _rewrites）；只有一边有的，不是恒为真的
|
||||
（_always）。aspect 是 join：说的是关联条件，dual_check_facts 不会把它当成业务日期写进数据目录。
|
||||
|
||||
只改说明，不碰等价判定：走到这里时两边已经判了不等价（见 explain_sql_diff）。"""
|
||||
from sqlglot import exp
|
||||
|
||||
if pa != pb or len(ja) != len(jb) or not self.visible:
|
||||
return None
|
||||
differing = [i for i, (x, y) in enumerate(zip(ja, jb)) if x != y]
|
||||
if len(differing) != 1:
|
||||
return None
|
||||
i = differing[0]
|
||||
if ja[i][:3] != jb[i][:3] or ja[i][2] != "left":
|
||||
return None
|
||||
alias = ja[i][0]
|
||||
join_a, join_b = A.joins[i], B.joins[i]
|
||||
rel = A.rel(alias)
|
||||
if rel is None or not rel.table or A.names().count(rel.name) != 1 or B.names().count(rel.name) != 1:
|
||||
return None
|
||||
on_a, on_b = _conjuncts(join_a.args["on"]), _conjuncts(join_b.args["on"])
|
||||
only_a, only_b = self._unmatched(on_a, on_b, {})
|
||||
keys: set[tuple[str, str]] = set()
|
||||
for block, conds in ((A, only_a), (B, only_b)):
|
||||
for cond in conds:
|
||||
cols = self.base_columns(cond, block)
|
||||
found = self.timed(cond, block)
|
||||
if found is None or not cols or any(r.alias != alias for r, _ in cols):
|
||||
return None
|
||||
keys.add(found)
|
||||
for block, join in ((A, join_a), (B, join_b)):
|
||||
if self._used_outside(block, alias, join, outputs=False):
|
||||
return None
|
||||
if not any(col.table == alias for e in block.exprs for col in e.find_all(exp.Column)):
|
||||
return None
|
||||
what = self.what(rel)
|
||||
if only_a and only_b:
|
||||
if len(keys) == 1:
|
||||
key = next(iter(keys))
|
||||
same_a = [c for c in on_a if self.refs(c, A) == {key}]
|
||||
same_b = [c for c in on_b if self.refs(c, B) == {key}]
|
||||
if self.range.compare(same_a, same_b, never_null=False) is not False:
|
||||
return None
|
||||
shown_a, shown_b = self.show_all(_by_bound(same_a), A), self.show_all(_by_bound(same_b), B)
|
||||
text = _say(self.la, "用 LEFT JOIN 关联", what, "时对", self.named(*key), "的条件是", shown_a, "，",
|
||||
self.lb, "是", shown_b)
|
||||
else:
|
||||
cols_a = {self.timed(c, A) for c in only_a}
|
||||
cols_b = {self.timed(c, B) for c in only_b}
|
||||
if len(cols_a) != 1 or len(cols_b) != 1:
|
||||
return None
|
||||
shown_a, shown_b = self.show_all(only_a, A), self.show_all(only_b, B)
|
||||
text = _say(self.la, "用 LEFT JOIN 关联", what, "时按", self.named(*next(iter(cols_a))), "筛时间，",
|
||||
self.lb, "按", self.named(*next(iter(cols_b))))
|
||||
self.add("join", shown_a, shown_b, text)
|
||||
return True, {}, set(), set()
|
||||
side, block, conds = (0, A, only_a) if only_a else (1, B, only_b)
|
||||
if self._always(conds, block) is not False:
|
||||
return None
|
||||
shown = self.show_all(_by_bound(conds), block)
|
||||
me, them = (self.la, self.lb) if side == 0 else (self.lb, self.la)
|
||||
self.add("join", shown if side == 0 else None, shown if side == 1 else None,
|
||||
_say(me, "用 LEFT JOIN 关联", what, "时只关联", shown, "的行，", them, "没有这个条件"))
|
||||
return True, {}, set(), set()
|
||||
|
||||
def _swapped(self, A: _Block, B: _Block, x: _Rel, y: _Rel) -> tuple[bool, dict[str, str], set[str], set[str]] | None:
|
||||
"""换了一张表：开头那组来源只差一张实表（各自只出现一次、名字不只差在大小写上，都没带库名）。换了表的那一边，
|
||||
列照它自己的表说。"""
|
||||
@@ -8727,7 +8807,527 @@ def _column_map(sql: str, rendered: str, read: str | None, info: _Schema,
|
||||
return tuple({"index": i, "ours": a, "caliber": b} for i, (a, b) in enumerate(zip(ours, theirs)))
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 按引用的列投影（next-round 口径账本：整题等价按答案引用的那几列比）
|
||||
# ==========================================================================
|
||||
#
|
||||
# 真实的助手写 SQL 总会多选几列（订单数、日期常量、MIN / MAX、COALESCE……），报告只引用其中一列；口径的取数也只选它的
|
||||
# 指标要的那几列。拿整条 SQL 去比永远不等价，「上周的销售额是多少」就永远只标相关口径。这里把最外层 SELECT 列表里没被
|
||||
# 引用、去掉也不改变被引用那几列取值的选择项去掉（需要时也调换留下的几项的先后），得到一条改写后的 SQL，交给调用方再走
|
||||
# 现有的绑定和判等价（match_template → explain_sql_diff）：投影本身不判等价、不给信任，判等价的那一步一点不放宽。
|
||||
#
|
||||
# 只在确定去掉不影响时才投影，不然照实给不投影的原因（Projection.refused），调用方照旧整条比：
|
||||
# - 整条语句就是一段 SELECT（可以带 WITH）：UNION 这类合并、外面套着括号的不投影——各支的列按位置合并，去掉一支的一列
|
||||
# 就错位了；
|
||||
# - SELECT 里只有 WITH、选择列表、FROM、JOIN、WHERE、GROUP BY、HAVING、QUALIFY、WINDOW、ORDER BY、LIMIT、OFFSET；
|
||||
# DISTINCT / DISTINCT ON（按整行去重，去掉一列行数就可能变）和别的修饰（提示、INTO、加锁、采样、LATERAL VIEW……）
|
||||
# 不投影；
|
||||
# - 选择列表里没有 * / t.*（展开之后按位置对不准）；给了查询结果的列数（width）时要和选择列表的项数一样：被引用的列按
|
||||
# 位置对上选择项；
|
||||
# - ORDER BY / GROUP BY 里没有按位置引用的（ORDER BY 2：去掉、调换之后指到别的列）；
|
||||
# - 有 LIMIT / OFFSET 时只投影不分组的聚合（只出一行，取哪一行和选出来的列无关）：别的「取前几行」有并列时取到哪几行
|
||||
# 可能跟着选择列表变；
|
||||
# - 去掉的每一项：不含认不准会不会改变行数的函数（UNNEST / EXPLODE / generate_series 这类展开成多行的、sqlglot 不认识的
|
||||
# 函数）；它的输出名（别名，没有别名的是列名）在整条语句别处（同一段的 WHERE、GROUP BY、HAVING、QUALIFY、ORDER BY、
|
||||
# 窗口、别的选择项、子查询）一次都没有被不带表名地写到——SQLite 的 WHERE、GROUP BY 都认列别名，和表列同名的也算，
|
||||
# 宁可不投影；不是常量的话，同样的写法没有出现在 GROUP BY、HAVING、ORDER BY、QUALIFY、窗口里（「出现在 GROUP BY 里」
|
||||
# 「被 HAVING 用到」都在这一条）；
|
||||
# - 去掉之后这一段聚不聚合不变：没有 GROUP BY、HAVING 时去掉的不能是仅有的聚合（剩下的列就不止一行了）；留下的列里没有
|
||||
# 既不在聚合里、也不是分组键的列（SQLite 取 MAX 那一行的值、MySQL 取任意一行：去掉别的聚合可能换了取值的那一行）；
|
||||
# - 调换先后时，留下的各项不按名字写到别的选择项的输出名（DuckDB 这类认「前面的列别名」）。
|
||||
#
|
||||
# 改写在原文上做：按方言切词、按括号深度认出最外层 SELECT 列表每一项在原文里的起止，只把留下的几项按新的先后原样拼回去，
|
||||
# 选择列表前后的原文一字不动（explain_sql_diff 的保真守卫照样看得到原文里的写法）。拼完重新解析，语法树要和「原来的树换成
|
||||
# 留下的几项」一模一样，不然不投影。口径模板（{{ input.x }}）按 _template_slots 的哨兵渲染之后认起止，再按位置换回模板
|
||||
# 的原文；MySQL 原文里有被当成两个减号的 --（_mysql_dashes 会改写的）不投影：改写后位置对不上原文。
|
||||
|
||||
#: 投影的那一段能有的子句
|
||||
_PROJECTABLE_ARGS = frozenset({"with_", "expressions", "from_", "joins", "where", "group", "having", "qualify",
|
||||
"windows", "order", "limit", "offset"})
|
||||
#: 去掉的选择项里不许有的、可能把一行展开成多行的函数（sqlglot 的类名；表函数的基类 UDTF、不认识的函数另认）
|
||||
_ROW_CHANGING = frozenset({"Explode", "Posexplode", "ExplodeOuter", "PosexplodeOuter", "Unnest", "Inline", "Stack",
|
||||
"ExplodingGenerateSeries", "GenerateSeries"})
|
||||
#: 最外层 SELECT 列表到这些词就结束了
|
||||
_LIST_ENDS = _CLAUSE_ENDS | {"FROM", "INTO"}
|
||||
_OPENS = frozenset({"L_PAREN", "L_BRACKET", "L_BRACE"})
|
||||
_CLOSES = frozenset({"R_PAREN", "R_BRACKET", "R_BRACE"})
|
||||
|
||||
#: 不投影的原因（只进比对的说明和测试，不判等价）
|
||||
_PROJ_UNPARSED = "SQL 无法解析，不投影"
|
||||
_PROJ_SETOP = "不是一段 SELECT（UNION 这类合并、外面套着括号），不投影"
|
||||
_PROJ_DISTINCT = "有 DISTINCT，去掉一列行数就可能变，不投影"
|
||||
_PROJ_CLAUSE = "SELECT 里有投影认不准的子句（{clause}），不投影"
|
||||
_PROJ_STAR = "选择列表里有 *，按位置对不准，不投影"
|
||||
_PROJ_WIDTH = "查询结果的列数（{width}）和选择列表的项数（{items}）对不上，不投影"
|
||||
_PROJ_KEEP = "要留下的列不对（空的、重复的、超出选择列表），不投影"
|
||||
_PROJ_ORDINAL = "ORDER BY / GROUP BY 按位置引用了选择项，去掉、调换之后会指到别的列，不投影"
|
||||
_PROJ_LIMIT = "有 LIMIT / OFFSET，取哪几行可能跟着选择列表变，不投影"
|
||||
_PROJ_ROWS = "去掉的「{item}」里有可能改变行数的函数，不投影"
|
||||
_PROJ_NAMED = "去掉的「{item}」在别处被按名字写到了，不投影"
|
||||
_PROJ_USED = "去掉的「{item}」同样的写法出现在 GROUP BY、HAVING、ORDER BY、QUALIFY 或窗口里，不投影"
|
||||
_PROJ_AGGREGATE = "去掉的是仅有的聚合，剩下的列就不止一行了，不投影"
|
||||
_PROJ_LOOSE = "留下的列里有既不在聚合里、也不是分组键的列，去掉别的列可能换了取值的那一行，不投影"
|
||||
_PROJ_REORDER = "留下的「{item}」按名字写到了别的选择项，不调换先后"
|
||||
_PROJ_TEXT = "认不出选择列表在原文里的起止，不投影"
|
||||
_PROJ_DASHES = "原文里有 MySQL 当成两个减号的 --，不投影"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Projection:
|
||||
"""project_select / project_template 的结果。
|
||||
|
||||
sql 是投影后的 SQL（模板投影之后仍是模板，{{ input.x }} 原样留着）；kept 是留下的选择项在原来的选择列表里的位置
|
||||
（按投影后的先后）；dropped 是去掉了几项；names 是留下的各项的输出名（别名、列名，别的写成它的 SQL，同
|
||||
_result_columns）。refused 不为空时不投影：sql 为 None，写为什么。要留下的就是全部、先后也没变时原样交回（dropped
|
||||
为 0）。"""
|
||||
|
||||
sql: str | None
|
||||
kept: tuple[int, ...] = ()
|
||||
dropped: int = 0
|
||||
names: tuple[str, ...] = ()
|
||||
refused: str | None = None
|
||||
|
||||
|
||||
class _NotProjectable(Exception):
|
||||
"""不投影：args[0] 是原因。"""
|
||||
|
||||
|
||||
def _output_name(item: Any, read: str | None) -> str:
|
||||
from sqlglot import exp
|
||||
|
||||
name = item.alias_or_name if isinstance(item, (exp.Alias, exp.Column)) else ""
|
||||
return name or item.sql(dialect=read)
|
||||
|
||||
|
||||
def _name_of(item: Any) -> str:
|
||||
"""选择项按名字被引用时的那个名字（别名、列名；小写）。表达式、常量没有。"""
|
||||
from sqlglot import exp
|
||||
|
||||
return (item.alias_or_name if isinstance(item, (exp.Alias, exp.Column)) else "").translate(_ASCII_LOWER)
|
||||
|
||||
|
||||
def _ordinal(node: Any) -> bool:
|
||||
"""ORDER BY / GROUP BY 的一项是按位置引用（一个整数常量）。"""
|
||||
from sqlglot import exp
|
||||
|
||||
inner = _unparen(node.this if isinstance(node, exp.Ordered) else node)
|
||||
return isinstance(inner, exp.Literal) and not inner.is_string
|
||||
|
||||
|
||||
def _projection_problem(root: Any, keep: Sequence[int], read: str | None) -> str | None:
|
||||
"""按 keep 投影这一段会不会改变留下的那几列的取值（见本节开头）：会、或者认不准时交回原因，确定不会时为 None。"""
|
||||
from sqlglot import exp
|
||||
|
||||
if not isinstance(root, exp.Select):
|
||||
return _PROJ_SETOP
|
||||
if root.args.get("distinct") is not None:
|
||||
return _PROJ_DISTINCT
|
||||
extra = sorted(k for k, v in root.args.items() if v not in (None, False, []) and k not in _PROJECTABLE_ARGS)
|
||||
if extra:
|
||||
return _PROJ_CLAUSE.format(clause=extra[0])
|
||||
items = list(root.expressions)
|
||||
if any(_is_star(item) for item in items):
|
||||
return _PROJ_STAR
|
||||
if not keep or len(set(keep)) != len(keep) or any(not isinstance(k, int) or not 0 <= k < len(items) for k in keep):
|
||||
return _PROJ_KEEP
|
||||
group, order = root.args.get("group"), root.args.get("order")
|
||||
for clause in (group, order):
|
||||
if clause is not None and any(_ordinal(e) for e in clause.expressions):
|
||||
return _PROJ_ORDINAL
|
||||
if (root.args.get("limit") is not None or root.args.get("offset") is not None) and \
|
||||
not (group is None and _aggregating(root)):
|
||||
return _PROJ_LIMIT
|
||||
kept = set(keep)
|
||||
removed = [i for i in range(len(items)) if i not in kept]
|
||||
holders = [c for c in (group, root.args.get("having"), order, root.args.get("qualify")) if c is not None]
|
||||
holders += [w for w in root.args.get("windows") or [] if isinstance(w, exp.Expr)]
|
||||
holders += [w for k in keep for w in items[k].find_all(exp.Window)]
|
||||
used = {_fingerprint(n) for holder in holders for n in holder.walk()}
|
||||
for i in removed:
|
||||
item = items[i]
|
||||
shown = item.sql(dialect=read, comments=False)[:60]
|
||||
if any(isinstance(n, (exp.UDTF, exp.Anonymous)) or type(n).__name__ in _ROW_CHANGING for n in item.walk()):
|
||||
return _PROJ_ROWS.format(item=shown)
|
||||
name = _name_of(item)
|
||||
if name:
|
||||
inside = {id(n) for n in item.walk()}
|
||||
if any(isinstance(n, exp.Column) and not n.table and id(n) not in inside
|
||||
and n.name.translate(_ASCII_LOWER) == name for n in root.walk()):
|
||||
return _PROJ_NAMED.format(item=shown)
|
||||
body = item.unalias()
|
||||
if not _constant(body) and _fingerprint(body) in used:
|
||||
return _PROJ_USED.format(item=shown)
|
||||
projected = root.copy()
|
||||
projected.set("expressions", [items[k].copy() for k in keep])
|
||||
if group is None and root.args.get("having") is None and _aggregating(root) != _aggregating(projected):
|
||||
return _PROJ_AGGREGATE
|
||||
if _loose_columns(projected):
|
||||
return _PROJ_LOOSE
|
||||
if list(keep) != sorted(keep):
|
||||
names = {_name_of(item) for item in items} - {""}
|
||||
for k in keep:
|
||||
others = names - {_name_of(items[k])}
|
||||
if any(isinstance(n, exp.Column) and not n.table and n.name.translate(_ASCII_LOWER) in others
|
||||
for n in items[k].walk()):
|
||||
return _PROJ_REORDER.format(item=items[k].sql(dialect=read, comments=False)[:60])
|
||||
return None
|
||||
|
||||
|
||||
def _list_spans(text: str, read: str | None) -> list[tuple[int, int]] | None:
|
||||
"""最外层 SELECT 列表每一项在原文里的起止（含两头）：按方言切词、按括号深度，深度 0 的 SELECT 之后、深度 0 的子句
|
||||
开头的词（FROM、WHERE……）之前，深度 0 的逗号分开。深度 0 的 SELECT 不止一个、括号对不上、有空项的为 None。"""
|
||||
try:
|
||||
tokens = _tokens(text, read)
|
||||
except Exception: # noqa: BLE001 - 切词都不行：认不出
|
||||
return None
|
||||
depth = 0
|
||||
state = 0 # 0：还没到最外层的 SELECT；1：在选择列表里；2：选择列表之后
|
||||
spans: list[tuple[int, int]] = []
|
||||
start: int | None = None
|
||||
end = 0
|
||||
for token in tokens:
|
||||
kind = token.token_type.name
|
||||
if state == 1 and depth == 0 and (kind == "COMMA" or kind in _LIST_ENDS):
|
||||
if start is None:
|
||||
return None
|
||||
spans.append((start, end))
|
||||
start = None
|
||||
if kind != "COMMA":
|
||||
state = 2
|
||||
continue
|
||||
if kind in _OPENS:
|
||||
depth += 1
|
||||
elif kind in _CLOSES:
|
||||
depth -= 1
|
||||
if depth < 0:
|
||||
return None
|
||||
elif depth == 0 and kind == "SELECT":
|
||||
if state != 0:
|
||||
return None
|
||||
state = 1
|
||||
continue
|
||||
if state == 1:
|
||||
start = token.start if start is None else start
|
||||
end = token.end
|
||||
if state == 1:
|
||||
if start is None:
|
||||
return None
|
||||
spans.append((start, end))
|
||||
return spans or None
|
||||
|
||||
|
||||
def _cut(text: str, spans: Sequence[tuple[int, int]], keep: Sequence[int]) -> str:
|
||||
"""把选择列表换成留下的几项（按 keep 的先后，原文照抄、逗号隔开）：选择列表前后的原文一字不动。"""
|
||||
head, tail = text[:spans[0][0]], text[spans[-1][1] + 1:]
|
||||
return head + ", ".join(text[spans[k][0]:spans[k][1] + 1] for k in keep) + tail
|
||||
|
||||
|
||||
def _plan_projection(text: str, keep: Sequence[int], read: str | None, width: int | None
|
||||
) -> tuple[Any, list[tuple[int, int]] | None, Any]:
|
||||
"""(原文的语法树, 选择列表各项的起止, 投影后应有的语法树)。原样（keep 就是全部、先后不变）时后两样为 None。不投影抛
|
||||
_NotProjectable。"""
|
||||
if read == "mysql" and _mysql_dashes(text) != text:
|
||||
raise _NotProjectable(_PROJ_DASHES)
|
||||
root, _ = _parse(text, read)
|
||||
if root is None:
|
||||
raise _NotProjectable(_PROJ_UNPARSED)
|
||||
from sqlglot import exp
|
||||
|
||||
items = list(root.expressions) if isinstance(root, exp.Select) else []
|
||||
if width is not None and isinstance(root, exp.Select) and len(items) != width:
|
||||
raise _NotProjectable(_PROJ_WIDTH.format(width=width, items=len(items)))
|
||||
if isinstance(root, exp.Select) and list(keep) == list(range(len(items))):
|
||||
return root, None, None
|
||||
why = _projection_problem(root, keep, read)
|
||||
if why:
|
||||
raise _NotProjectable(why)
|
||||
spans = _list_spans(text, read)
|
||||
if spans is None or len(spans) != len(items):
|
||||
raise _NotProjectable(_PROJ_TEXT)
|
||||
expected = root.copy()
|
||||
expected.set("expressions", [items[k].copy() for k in keep])
|
||||
return root, spans, expected
|
||||
|
||||
|
||||
def _same_tree(text: str, read: str | None, expected: Any) -> bool:
|
||||
root, _ = _parse(text, read)
|
||||
return root is not None and _fingerprint(root) == _fingerprint(expected)
|
||||
|
||||
|
||||
def project_select(sql: str, keep: Sequence[int], dialect: str | None = None, *,
|
||||
width: int | None = None) -> Projection:
|
||||
"""按引用的列投影一条查询（见本节开头）：最外层选择列表只留 keep 那几项（位置从 0 数，按 keep 的先后）。从不抛异常。
|
||||
|
||||
width 是查询结果的列数（查询快照里记的）：给了就要和选择列表的项数一样，被引用的列才按位置对得上选择项。不投影时
|
||||
refused 写为什么，调用方照旧整条比。投影出来的 SQL 不判等价：交给 match_template / explain_sql_diff 判。"""
|
||||
read = _read(dialect)
|
||||
text = (sql or "").strip()
|
||||
keep = tuple(keep)
|
||||
try:
|
||||
root, spans, expected = _plan_projection(text, keep, read, width)
|
||||
items = list(root.expressions)
|
||||
names = tuple(_output_name(items[k], read) for k in keep)
|
||||
if spans is None:
|
||||
return Projection(text, keep, 0, names)
|
||||
out = _cut(text, spans, keep)
|
||||
if not _same_tree(out, read, expected):
|
||||
raise _NotProjectable(_PROJ_TEXT)
|
||||
return Projection(out, keep, len(items) - len(keep), names)
|
||||
except _NotProjectable as e:
|
||||
return Projection(None, keep, refused=str(e))
|
||||
except Exception: # noqa: BLE001 - 投影本身的缺陷不能连累调用方：当作不投影，照旧整条比
|
||||
logger.warning("SQL 投影失败", exc_info=True)
|
||||
return Projection(None, keep, refused=_PROJ_TEXT)
|
||||
|
||||
|
||||
def _template_marks(template: str, spans: Sequence[tuple[int, int, TemplateSlot]]) -> list[tuple[int, int, int, int]]:
|
||||
"""模板里每一处槽位在哨兵渲染的 SQL 里、在模板原文里的起止：[(哨兵起, 哨兵止, 模板起, 模板止)]（止不含）。槽位之外
|
||||
两边逐字相同（_template_slots 只换槽位）。"""
|
||||
out: list[tuple[int, int, int, int]] = []
|
||||
shift = 0
|
||||
for start, end, slot in spans:
|
||||
size = len(_STRING_SENTINEL.format(slot.index)) if slot.context == "quoted" else \
|
||||
len(str(_NUMBER_SENTINEL + slot.index)) if slot.context == "bare" else 0
|
||||
out.append((start - shift, start - shift + size, start, end))
|
||||
shift += (end - start) - size
|
||||
return out
|
||||
|
||||
|
||||
def _to_template(pos: int, marks: Sequence[tuple[int, int, int, int]], *, last: bool) -> int | None:
|
||||
"""哨兵 SQL 里的一个位置 → 模板原文里的位置。落在哨兵中间的认不出（选择项的起止只会在一个词的两头）。last：这是一项
|
||||
的最后一个字（含），落在哨兵最后一个字上时对到槽位的最后一个字。"""
|
||||
delta = 0
|
||||
for s0, s1, t0, t1 in marks:
|
||||
if pos < s0:
|
||||
break
|
||||
if pos < s1:
|
||||
if not last and pos == s0:
|
||||
return t0
|
||||
if last and pos == s1 - 1:
|
||||
return t1 - 1
|
||||
return None
|
||||
delta = t1 - s1
|
||||
return pos + delta
|
||||
|
||||
|
||||
def project_template(template_sql: str, keep: Sequence[int], dialect: str | None = None) -> Projection:
|
||||
"""按口径卡读到的列投影口径的取数模板（{{ input.<字段> }} 原样留着，见本节开头）。规则同 project_select，在哨兵渲染
|
||||
的 SQL 上判；起止按位置换回模板原文。从不抛异常。"""
|
||||
read = _read(dialect)
|
||||
template = (template_sql or "").strip()
|
||||
keep = tuple(keep)
|
||||
try:
|
||||
slots, sentinel = _template_slots(template)
|
||||
root, spans, expected = _plan_projection(sentinel, keep, read, None)
|
||||
items = list(root.expressions)
|
||||
names = tuple(_output_name(items[k], read) for k in keep)
|
||||
if spans is None:
|
||||
return Projection(template, keep, 0, names)
|
||||
if not _same_tree(_cut(sentinel, spans, keep), read, expected):
|
||||
raise _NotProjectable(_PROJ_TEXT)
|
||||
marks = _template_marks(template, slots)
|
||||
mapped = [(_to_template(s, marks, last=False), _to_template(e, marks, last=True)) for s, e in spans]
|
||||
if any(s is None or e is None for s, e in mapped):
|
||||
raise _NotProjectable(_PROJ_TEXT)
|
||||
out = _cut(template, mapped, keep) # type: ignore[arg-type]
|
||||
# 换回模板之后再按哨兵渲染一遍：结构（字面量只看位置，槽位的编号会因为去掉了几处而变）要和投影后的一样
|
||||
_, again = _template_slots(out)
|
||||
root_again, _ = _parse(again, read)
|
||||
lits_a: list[Any] = []
|
||||
lits_b: list[Any] = []
|
||||
if root_again is None or _erased(root_again, lits_a) != _erased(expected, lits_b) or len(lits_a) != len(lits_b):
|
||||
raise _NotProjectable(_PROJ_TEXT)
|
||||
return Projection(out, keep, len(items) - len(keep), names)
|
||||
except _NotComparable as e:
|
||||
return Projection(None, keep, refused=str(e))
|
||||
except _NotProjectable as e:
|
||||
return Projection(None, keep, refused=str(e))
|
||||
except Exception: # noqa: BLE001 - 同 project_select
|
||||
logger.warning("口径模板投影失败", exc_info=True)
|
||||
return Projection(None, keep, refused=_PROJ_TEXT)
|
||||
|
||||
|
||||
def output_names(sql: str, dialect: str | None = None, *, template: bool = False) -> list[str] | None:
|
||||
"""最外层选择列表各项的输出名（按位置；同 _result_columns，不展开 *：有 * 的为 None）。template 为真时 sql 是口径
|
||||
模板，按哨兵渲染之后认。解析不了、不是一段 SELECT 的为 None。"""
|
||||
from sqlglot import exp
|
||||
|
||||
read = _read(dialect)
|
||||
text = (sql or "").strip()
|
||||
try:
|
||||
if template:
|
||||
_, text = _template_slots(text)
|
||||
root, _ = _parse(_mysql_dashes(text) if read == "mysql" else text, read)
|
||||
except _NotComparable:
|
||||
return None
|
||||
if not isinstance(root, exp.Select) or any(_is_star(i) for i in root.expressions):
|
||||
return None
|
||||
return [_output_name(i, read) for i in root.expressions]
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 同一个形状（认证队列的「确定性重复」：条件的先后不算）
|
||||
# ==========================================================================
|
||||
#
|
||||
# normalize_sql 的规范文本把字面量换成「?」，但条件照原文的先后：WHERE a = 1 AND b = 2 和 WHERE b = 2 AND a = 1 文本不同，
|
||||
# 认证队列就拆成两组。这里在不认列的那一级规范形式（_light，和判等价同一套只保持语义的改写）上，把字面量抹成只看类型的
|
||||
# 占位，再把交换律成立的地方按抹掉之后的写法重排一遍：AND / OR 的各项、IN 列表、= / <> / + / * 的两边、比大小的两边（换边
|
||||
# 时大于小于跟着换）。规范形式里本来的排序看字面量的值（同一个形状换了参数，条件的先后可能就变了），所以要重排。重排时
|
||||
# 抹掉之后一样的几项按原值排：同一组里字面量的位置对得上。认不出的（解析不了、sqlglot 会读错的写法、规范化不了的结构）
|
||||
# 为 None，调用方照旧按 normalize_sql 的文本分组：不同就是不同，只会分得细，不会把不同的并成一组。
|
||||
|
||||
#: 抹掉字面量之前给每个字面量记的编号（对回 normalize_sql 的字面量用）
|
||||
_LIT_TAG = "sql_semantics_literal"
|
||||
#: 对 normalize_sql 的先后时，数字字面量换成的标记：这个整数加编号（同 _NUMBER_SENTINEL，真实 SQL 里几乎不会写）
|
||||
_LIT_NUMBER = 9_173_402_581_000
|
||||
_LIT_STRING = "__agl_l{}__"
|
||||
_LIT_STRING_RE = re.compile(r"^'__agl_l(\d+)__'$")
|
||||
#: 交换律成立、两边可以对调的运算
|
||||
_SYMMETRIC = ("EQ", "NEQ", "Add", "Mul", "NullSafeEQ")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SqlShape:
|
||||
"""sql_shape 的结果。key 一样就是同一个形状（只差字面量的值、条件的先后、交换律成立处两边的先后）；literals 是按规范
|
||||
形式里的先后排的字面量（SQL 写法，字符串带引号），同一个形状的几条按位置对得上；positions[i] 是第 i 个在
|
||||
normalize_sql(sql).literals 里的位置（规范化时新算出来的字面量对不上，为 None）。"""
|
||||
|
||||
key: str
|
||||
literals: tuple[str, ...]
|
||||
positions: tuple[int | None, ...]
|
||||
|
||||
|
||||
def _normalized_order(root: Any, read: str | None) -> list[int] | None:
|
||||
"""normalize_sql 的字面量（按它重新生成的文本里的先后）各是哪一个字面量节点（_LIT_TAG 的编号）：把字面量换成带编号的
|
||||
标记，同样地重新生成、切词，按先后读出编号。对不上（个数不同、有认不出的）为 None。"""
|
||||
from sqlglot import exp
|
||||
from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
|
||||
|
||||
marked = normalize_identifiers(root.copy(), dialect=read)
|
||||
for lit in marked.find_all(exp.Literal):
|
||||
tag = lit.meta.get(_LIT_TAG)
|
||||
if tag is None:
|
||||
return None
|
||||
lit.set("this", _LIT_STRING.format(tag) if lit.is_string else str(_LIT_NUMBER + tag))
|
||||
text = marked.sql(dialect=read, comments=False)
|
||||
out: list[int] = []
|
||||
for token in _tokens(text, read):
|
||||
if not _is_literal(token):
|
||||
continue
|
||||
raw = text[token.start:token.end + 1]
|
||||
found = _LIT_STRING_RE.match(raw)
|
||||
if found:
|
||||
out.append(int(found.group(1)))
|
||||
continue
|
||||
try:
|
||||
out.append(int(raw) - _LIT_NUMBER)
|
||||
except ValueError:
|
||||
return None
|
||||
return out
|
||||
|
||||
|
||||
def _shape_print(node: Any, read: str | None) -> tuple[str, list[Any]]:
|
||||
"""规范形式的确定写法（同 _erased：字面量只写类型、不写值，按写出的先后收进列表），交换律成立的地方按抹掉之后的写法
|
||||
重排（抹掉之后一样的几项按原值排）。"""
|
||||
from sqlglot import exp
|
||||
|
||||
def value_key(lits: list[Any]) -> tuple[str, ...]:
|
||||
return tuple(lit.sql(dialect=read) for lit in lits)
|
||||
|
||||
def ordered(parts: list[tuple[str, list[Any]]]) -> list[tuple[str, list[Any]]]:
|
||||
return sorted(parts, key=lambda p: (p[0], value_key(p[1])))
|
||||
|
||||
if isinstance(node, exp.Paren):
|
||||
return _shape_print(node.this, read)
|
||||
if isinstance(node, exp.Literal):
|
||||
return f"Literal({'s' if node.is_string else 'n'})", [node]
|
||||
if isinstance(node, (exp.And, exp.Or)):
|
||||
parts = ordered([_shape_print(p, read) for p in _flat(node, type(node))])
|
||||
return (f"{type(node).__name__}[{','.join(p[0] for p in parts)}]",
|
||||
[lit for _, lits in parts for lit in lits])
|
||||
if isinstance(node, exp.Expr) and type(node).__name__ in _FLIP and not _quantified(node.this) \
|
||||
and not _quantified(node.expression):
|
||||
left, right = _shape_print(node.this, read), _shape_print(node.expression, read)
|
||||
name = type(node).__name__
|
||||
later = (left[0], value_key(left[1])) > (right[0], value_key(right[1]))
|
||||
fixed_left, fixed_right = _fixed(node.this), _fixed(node.expression)
|
||||
if (fixed_left and not fixed_right) or (fixed_left == fixed_right and later):
|
||||
left, right, name = right, left, _FLIP[name]
|
||||
return f"{name}({left[0]},{right[0]})", left[1] + right[1]
|
||||
if isinstance(node, exp.Expr) and type(node).__name__ in _SYMMETRIC:
|
||||
parts = ordered([_shape_print(node.this, read), _shape_print(node.expression, read)])
|
||||
return f"{type(node).__name__}({','.join(p[0] for p in parts)})", parts[0][1] + parts[1][1]
|
||||
if isinstance(node, exp.In) and node.args.get("expressions"):
|
||||
head = _shape_print(node.this, read)
|
||||
parts = ordered([_shape_print(e, read) for e in node.expressions])
|
||||
rest = [f"{k}={_shape_print(v, read)[0]}" for k, v in sorted(node.args.items())
|
||||
if k not in ("this", "expressions") and v is not None and v is not False and v != []]
|
||||
return (f"In({head[0]};[{','.join(p[0] for p in parts)}];{','.join(rest)})",
|
||||
head[1] + [lit for _, lits in parts for lit in lits])
|
||||
if isinstance(node, exp.Expr):
|
||||
pieces: list[str] = []
|
||||
lits: list[Any] = []
|
||||
for k, v in sorted(node.args.items()):
|
||||
if v is None or v is False or v == []:
|
||||
continue
|
||||
text, found = _shape_print(v, read)
|
||||
pieces.append(f"{k}={text}")
|
||||
lits += found
|
||||
mark = node.meta.get("sql_semantics_join") if isinstance(node, exp.Join) else None
|
||||
if mark:
|
||||
pieces.append(f"@join={mark}")
|
||||
surface = node.meta.get(_SURFACE)
|
||||
if surface:
|
||||
pieces.append(f"@surface={surface}")
|
||||
return f"{type(node).__name__}({','.join(pieces)})", lits
|
||||
if isinstance(node, list):
|
||||
parts = [_shape_print(v, read) for v in node]
|
||||
return "[" + ",".join(p[0] for p in parts) + "]", [lit for _, found in parts for lit in found]
|
||||
return repr(node), []
|
||||
|
||||
|
||||
def sql_shape(sql: str, dialect: str | None = None) -> SqlShape | None:
|
||||
"""一条查询的形状（见本节开头）：条件的先后、交换律成立处两边的先后、字面量的值都不算；别的不同一律算不同。认不出的为
|
||||
None（调用方按 normalize_sql 的文本分组）。从不抛异常。"""
|
||||
from sqlglot import exp
|
||||
|
||||
read = _read(dialect)
|
||||
text = (_mysql_dashes(sql) if read == "mysql" and isinstance(sql, str) else sql or "").strip()
|
||||
root, _ = _parse(text, read)
|
||||
if root is None:
|
||||
return None
|
||||
try:
|
||||
if _lost_in_parsing(text, read, root) or _respelled(text, read, root):
|
||||
return None
|
||||
for i, lit in enumerate(root.find_all(exp.Literal)):
|
||||
lit.meta[_LIT_TAG] = i
|
||||
order = _normalized_order(root, read)
|
||||
if order is None or len(order) != len(normalize_sql(sql, dialect).literals):
|
||||
return None
|
||||
marked = _mark_surfaces(root, text, read)
|
||||
if read == "sqlite":
|
||||
_mark_comma_joins(root, text, read)
|
||||
if not marked:
|
||||
return None
|
||||
try:
|
||||
tree = _light(root, read, _Schema()).tree
|
||||
except _Unsupported:
|
||||
# 规范化不了的结构（递归 CTE、不确定的函数……）：只统一名字，条件的先后照原文
|
||||
tree = root.copy()
|
||||
_normalize_names(tree, read, _Schema())
|
||||
printed, lits = _shape_print(tree, read)
|
||||
except Exception: # noqa: BLE001 - 认形状本身的缺陷：当作认不出，调用方按文本分组
|
||||
logger.warning("认 SQL 的形状失败", exc_info=True)
|
||||
return None
|
||||
where = {tag: i for i, tag in enumerate(order)}
|
||||
return SqlShape(hashlib.sha256(printed.encode()).hexdigest(),
|
||||
tuple(lit.sql(dialect=read) for lit in lits),
|
||||
tuple(where.get(lit.meta.get(_LIT_TAG)) if lit.meta.get(_LIT_TAG) is not None else None
|
||||
for lit in lits))
|
||||
|
||||
|
||||
__all__ = ["BINDING_LEVELS", "MAX_SQL_CHARS", "PLACEHOLDER", "Aggregate", "ColumnRef", "Computed", "FacetDiff", "Facets",
|
||||
"JoinFacet", "LimitFacet", "NormalizedSql", "Part", "SqlDiff", "TemplateBinding", "TemplateMatch",
|
||||
"TemplateSlot", "TimePredicate", "bind_template", "catalog_labels", "diff_facets", "explain_sql_diff",
|
||||
"facets", "match_template", "normalize_sql", "render_bound"]
|
||||
"JoinFacet", "LimitFacet", "NormalizedSql", "Part", "Projection", "SqlDiff", "SqlShape", "TemplateBinding",
|
||||
"TemplateMatch", "TemplateSlot", "TimePredicate", "bind_template", "catalog_labels", "diff_facets",
|
||||
"explain_sql_diff", "facets", "match_template", "normalize_sql", "output_names", "project_select",
|
||||
"project_template", "render_bound", "sql_shape"]
|
||||
|
||||
@@ -6,7 +6,9 @@
|
||||
|
||||
- 运行时（nodes/metrics._input_of）：「只是把调用工具节点的结果原样按 JSON 解析」的数据整形节点（parsed_tool）不改动任何
|
||||
一格，vars.<x>.rows[i][j]（row_cell）确定地对到那份查询快照的第 i 行第 j 列，和 cell() 一样按快照核对，记下穿过的整形
|
||||
节点（through）；
|
||||
节点（through）；转抄另一张口径卡的指标（nodes.<卡>.metrics[i].value，metric_copy：对账卡、召回卡的「口径」那一项就是
|
||||
这么写的）跨卡往下追——接上那个指标自己的出处，那个指标的输入都追到了原始数据，这一项才算追到（upstream.traced）；
|
||||
转抄的转抄一层层接着追，最多 MAX_COPY_DEPTH 层（循环里抄自己上一轮的卡每一轮多一层，到了上限就算没追到）；
|
||||
- 证据面板（api/evidence 的指标步骤）：卡上记的输入里追不到原始数据的（untraced），照实说出处追到哪个节点为止；
|
||||
- 应用检查（apps/lint）：受管级应用里绑定的指标，按 static_stops 静态地认（只有图、没有运行），追不到的给提示。
|
||||
"""
|
||||
@@ -81,12 +83,39 @@ def row_cell(path: str) -> RowCell | None:
|
||||
return RowCell(rows.value.value.id, rows.value.attr, row.value, column.value)
|
||||
|
||||
|
||||
#: 转抄另一张口径卡的指标时跨卡往下追的层数上限
|
||||
MAX_COPY_DEPTH = 8
|
||||
|
||||
|
||||
def metric_copy(path: str) -> tuple[str, str, int] | None:
|
||||
"""路径正好是 nodes.<卡>.metrics[i].value、vars.<卡的 assign_to>.metrics[i].value 这种转抄一个指标的值的样子时拆开：
|
||||
(root, 名字, 第几个指标)。下标是非负整数字面量；别的样子（整张卡、清单文字、指标的名称单位、下标是变量）为 None。"""
|
||||
try:
|
||||
tree = ast.parse(path, mode="eval").body
|
||||
except SyntaxError:
|
||||
return None
|
||||
if not (isinstance(tree, ast.Attribute) and tree.attr == "value" and isinstance(tree.value, ast.Subscript)):
|
||||
return None
|
||||
index, metrics = tree.value.slice, tree.value.value
|
||||
if not (isinstance(metrics, ast.Attribute) and metrics.attr == "metrics"
|
||||
and isinstance(metrics.value, ast.Attribute) and isinstance(metrics.value.value, ast.Name)
|
||||
and metrics.value.value.id in ("vars", "nodes")):
|
||||
return None
|
||||
if not (isinstance(index, ast.Constant) and type(index.value) is int and index.value >= 0):
|
||||
return None
|
||||
return metrics.value.value.id, metrics.value.attr, index.value
|
||||
|
||||
|
||||
def traced(item: Mapping[str, Any]) -> bool:
|
||||
"""卡上记的一个输入追没追到原始数据：落在查询快照的一格上、核对得了快照（cell() 取的格、穿过 JSON 解析按位置取的格、
|
||||
Agent 按出处交来的字段），或者是运行输入。"""
|
||||
Agent 按出处交来的字段），或者是运行输入；转抄另一张口径卡的指标的，那个指标的输入都追到了（运行时跨卡追好记在
|
||||
upstream 里：traced 为真、那张卡的指标集落了工件）。"""
|
||||
via = item.get("via")
|
||||
if via in SNAPSHOT_VIAS:
|
||||
return bool(item.get("artifact")) and item.get("status") in _CHECKED
|
||||
if via == "metric":
|
||||
up = item.get("upstream")
|
||||
return isinstance(up, Mapping) and up.get("traced") is True and bool(up.get("artifact"))
|
||||
return via == "input"
|
||||
|
||||
|
||||
|
||||
@@ -132,12 +132,18 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
decimals = display["decimals"]
|
||||
value = round_metric(value, decimals)
|
||||
substituted = substitute(tree, values)
|
||||
incomplete: dict[str, Any] = {}
|
||||
if use.sources:
|
||||
sources = list(use.sources.values())
|
||||
incomplete = {"incomplete": True, "incomplete_reason": _incomplete_reason(sources),
|
||||
"truncated_sources": sources}
|
||||
inputs = [_input_of(path, values[path], state, ctx, tctx) for path in paths]
|
||||
# 转抄另一张口径卡的指标：那个指标不完整、所依据的查询没通过 SQL 检查的，这一个照样不完整、存疑（出处是跨卡追下去
|
||||
# 的，判档也按追到底的结果，见 engine/lineage.metric_copy）
|
||||
carried = [m for item in inputs if (m := _copied_metric(item, tctx)) is not None]
|
||||
truncated = list(use.sources.values())
|
||||
for source in (s for m in carried if m.get("incomplete") for s in m.get("truncated_sources") or []):
|
||||
if isinstance(source, dict) and source not in truncated:
|
||||
truncated.append(source)
|
||||
incomplete: dict[str, Any] = {}
|
||||
if truncated:
|
||||
incomplete = {"incomplete": True, "incomplete_reason": _incomplete_reason(truncated),
|
||||
"truncated_sources": truncated}
|
||||
for path, item in zip(paths, inputs):
|
||||
# 容器型的 vars.* 输入（列表、字典）另存一份完整值：卡里只记了「[12 项]」这样的大概，事后推演
|
||||
# （engine/recompute.py）要拿完整的值重算。nodes.*、input.* 的完整值在那个节点封存的产出工件里，
|
||||
@@ -146,7 +152,9 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
if _container_var(path, values[path]) and (stored := await _store_value(path, values[path], ctx)):
|
||||
item["value_artifact"] = stored
|
||||
sql_check: dict[str, Any] = {}
|
||||
if failed := _failed_queries(inputs, list(use.sources.values()), paths, tctx, checked):
|
||||
copied_checks = [str(c.get("artifact")) for m in carried for c in m.get("sql_check_sources") or []
|
||||
if isinstance(c, dict) and c.get("artifact")]
|
||||
if failed := _failed_queries(inputs, truncated, paths, tctx, checked, extra=copied_checks):
|
||||
sql_check = {"sql_check_failed": True, "sql_check_reason": _sql_check_reason(failed),
|
||||
"sql_check_sources": [{"artifact": artifact, "codes": list(dict.fromkeys(
|
||||
code for p in found for code in p.codes))} for artifact, found in failed]}
|
||||
@@ -456,18 +464,20 @@ _LAST_STEP = re.compile(r"(?:\.[^.\[\]]+|\[[^\[\]]*\])$")
|
||||
|
||||
|
||||
def _failed_queries(inputs: list[dict[str, Any]], truncated: list[dict[str, Any]], paths: list[str],
|
||||
tctx: dict[str, Any], checked: dict[str, list[QueryProblem]],
|
||||
) -> list[tuple[str, list[QueryProblem]]]:
|
||||
tctx: dict[str, Any], checked: dict[str, list[QueryProblem]], *,
|
||||
extra: list[str] | tuple[str, ...] = ()) -> list[tuple[str, list[QueryProblem]]]:
|
||||
"""这个指标用到的查询里，SQL 检查有 error 的：[(查询快照工件 id, 没通过检查的查询)]。合并查询的快照对应它出问题的
|
||||
那几个输入（sql_problems.query_problems）。
|
||||
|
||||
用到了哪些查询，三处来历合起来认：输入的出处（cell() 取的格、Agent 经 cite_fields 交来的字段都带着快照 id）、
|
||||
用到了哪些查询，四处来历合起来认：输入的出处（cell() 取的格、Agent 经 cite_fields 交来的字段都带着快照 id）、
|
||||
截断跟踪记下的整组、取值链上的查询结果本身（数据整形解析过的 vars.res.rows[0][0]，往上找到 vars.res 那份
|
||||
带 artifact 的结果）。检查结果从快照里读，不信内存里那份：快照按哈希寻址，就是查询当时存下的那份。
|
||||
带 artifact 的结果），以及 extra：转抄的指标所依据、没通过检查的那几条（跨卡追下去的）。检查结果从快照里读，不信内存里那份：快照按哈希寻址，就是查询当时存下的那份。
|
||||
读不出来的快照不算（另有别的检查报快照的问题）。Agent 没开 cite_fields 时说不清数出自哪次查询，不算。
|
||||
"""
|
||||
artifacts = [item.get("artifact") for item in inputs] + [s.get("artifact") for s in truncated]
|
||||
artifacts += [_result_artifact(path, tctx) for path in paths if cell_parts(path) is None]
|
||||
# 转抄的指标所依据、没通过检查的查询（extra，跨卡追下去的那几条）
|
||||
artifacts += list(extra)
|
||||
out = []
|
||||
for artifact in dict.fromkeys(a for a in artifacts if isinstance(a, str) and a):
|
||||
if artifact not in checked:
|
||||
@@ -646,6 +656,10 @@ def _input_of(path: str, value: Any, state: GraphState, ctx: NodeContext,
|
||||
# 沙箱算出来的数喂给口径卡：记下它自认是「取数」还是「计算」，治理规则按它判
|
||||
entry["role"] = str(producer.config.get("evidence_role") or "compute")
|
||||
entry["status"] = "ok" if value is not None else "missing"
|
||||
if producer is not None and str(producer.type) == "metrics":
|
||||
# 转抄另一张口径卡的指标：跨卡往下追（engine/lineage.metric_copy）
|
||||
if (up := _upstream(path, tctx or template_context(state))) is not None:
|
||||
entry["upstream"] = up
|
||||
if producer is not None and str(producer.type) == "agent" and field:
|
||||
cited = _agent_field(state, producer.id, root if head else "", field)
|
||||
if cited is not None:
|
||||
@@ -653,6 +667,48 @@ def _input_of(path: str, value: Any, state: GraphState, ctx: NodeContext,
|
||||
return entry
|
||||
|
||||
|
||||
def _copied_card(path: str, tctx: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
"""转抄另一张口径卡一个指标的输入（engine/lineage.metric_copy 认的写法）：(那张卡这一次的产出, 那个指标)。不是这种
|
||||
写法、产出不是指标集、下标越界的为 None。"""
|
||||
from app.engine.lineage import metric_copy
|
||||
|
||||
found = metric_copy(path)
|
||||
if found is None:
|
||||
return None
|
||||
root, name, index = found
|
||||
card = _value_of(f"{root}.{name}", tctx)
|
||||
metrics = card.get("metrics") if isinstance(card, dict) and card.get("kind") == "metric_set" else None
|
||||
if not isinstance(metrics, list) or index >= len(metrics) or not isinstance(metrics[index], dict):
|
||||
return None
|
||||
return card, metrics[index]
|
||||
|
||||
|
||||
def _upstream(path: str, tctx: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""转抄另一张口径卡的指标时，跨卡往下追的出处：{node_id?, metric, artifact, depth, traced}。那张卡的指标集落了工件、
|
||||
那个指标的输入都追到了原始数据（engine/lineage.untraced 为空；它自己转抄的再往下一层层追过了，记在它的输入上）、
|
||||
层数没超过 MAX_COPY_DEPTH，才算追到。不是转抄指标的为 None（照旧停在那张卡）。"""
|
||||
from app.engine.lineage import MAX_COPY_DEPTH, untraced
|
||||
|
||||
found = _copied_card(path, tctx)
|
||||
if found is None:
|
||||
return None
|
||||
card, metric = found
|
||||
inputs = [i for i in metric.get("inputs") or [] if isinstance(i, dict)]
|
||||
depth = 1 + max([int((i.get("upstream") or {}).get("depth") or 0) for i in inputs
|
||||
if isinstance(i.get("upstream"), dict)] or [0])
|
||||
artifact = card.get("artifact") if isinstance(card.get("artifact"), str) else None
|
||||
return {"metric": metric.get("id"), "artifact": artifact, "depth": depth,
|
||||
"traced": bool(artifact) and depth <= MAX_COPY_DEPTH and not untraced(inputs)}
|
||||
|
||||
|
||||
def _copied_metric(item: dict[str, Any], tctx: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""卡上的一个输入转抄了另一张口径卡的哪个指标（这一次的产出里那一项）；不是转抄的为 None。"""
|
||||
if item.get("via") != "metric" or not isinstance(item.get("upstream"), dict):
|
||||
return None
|
||||
found = _copied_card(str(item.get("path") or ""), tctx)
|
||||
return found[1] if found is not None else None
|
||||
|
||||
|
||||
def _field_key(field: str) -> str:
|
||||
"""vars.kpi['gmv'] 和 vars.kpi.gmv 是同一个字段：下标写法换成点号。"""
|
||||
return re.sub(r"\[\s*['\"]([^'\"\]]+)['\"]\s*\]", r".\1", field).strip(".")
|
||||
|
||||
@@ -5,10 +5,15 @@
|
||||
|
||||
**认证队列**（GET /api/ledger/certify-queue，搭建者及以上）三种信号：
|
||||
- 确定性重复（shape）：问数据的探索运行里执行过、被报告引用（比对工件里标签记着的查询）、又没对上任何口径（none / related）
|
||||
的查询，按（数据源、normalize_sql 的规范文本）分组；出现在 ≥ REPEAT_RUNS 次运行、来自 ≥ REPEAT_CONVERSATIONS 段对话的进队。
|
||||
跨运行变化的字面量位置是参数候选，不变的是口径常量。展示的是规范文本（字面量都换成了「?」，不带取值）；
|
||||
的查询，按（数据源、规范形）分组：sql_semantics.sql_shape 在判等价同一套规范形式上抹掉字面量、把 AND / OR 的各项、IN 列表、
|
||||
交换律成立的比较两边重排——条件的先后不同、AND 两边换位的进同一组，条件真不同的不进；认不出规范形的按 normalize_sql 的
|
||||
规范文本分组（只会分得细）。出现在 ≥ REPEAT_RUNS 次运行、来自 ≥ REPEAT_CONVERSATIONS 段对话的进队。跨运行变化的字面量
|
||||
位置（按规范形里的先后对齐）是参数候选，不变的是口径常量。展示的是代表那条的规范文本（字面量都换成了「?」，不带取值）。
|
||||
这一项的 id 按（数据源、规范形）算；以前按规范文本记的「不是口径」照样认（组里任何一条的规范文本对得上就算）；
|
||||
- 问题聚类（cluster）：app/chat/questions 全站唯一的聚类（只取问题原文、不含 history、排除系统派生的运行），≥ REPEAT_RUNS
|
||||
个问题的类；
|
||||
个问题的类。这一项的 id 是类的 id（成员的稳定 id 排好取哈希：同一批问题两次读取一样）；「不是口径」记下当时的成员，之后
|
||||
这一类过半的成员是那时记下的就照样不进队（来了几句新问法 id 会变，按成员认就不失效）。以前按种子问题原文算的 id 记的
|
||||
照样认，记的成员读不懂的照实忽略；
|
||||
- 被反复用的答案（reused）：导出过 ≥ 2 次（report.exported 审计，第 2 波）、做成过应用（蓝图的来源是这次运行）、在画布里
|
||||
打开过或提取过（工作流的 origin.run_id 是这次运行）的轮次。
|
||||
每项给代表问题、轮次链接、（shape 的）SQL 形状和参数候选、有没有相关口径（可以对账的那份）、是不是已经固化过（frozen_into，
|
||||
@@ -90,22 +95,39 @@ def item_id(kind: str, key: str) -> str:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Shape:
|
||||
"""一条查询的形状：数据源、规范文本（字面量换成「?」）、字面量（SQL 写法，字符串带引号）、规范化失败的原因。"""
|
||||
"""一条查询的形状：数据源、规范文本（字面量换成「?」）、字面量（SQL 写法，字符串带引号）、规范化失败的原因。
|
||||
|
||||
key 是分组用的规范形（sql_semantics.sql_shape，「shape:」开头；认不出时退回规范文本，「text:」开头）；canonical 是按
|
||||
规范形里的先后排的字面量（同一组的几条按位置对得上，认哪些位置跨轮变了用它），positions[i] 是它在 literals 里的位置
|
||||
（规范化时新算出来的字面量为 None：没法写成模板参数）。"""
|
||||
|
||||
source: str
|
||||
text: str
|
||||
literals: tuple[str, ...]
|
||||
problem: str | None = None
|
||||
key: str = ""
|
||||
canonical: tuple[str, ...] = ()
|
||||
positions: tuple[int | None, ...] = ()
|
||||
|
||||
def params(self, canonical_positions: Sequence[int]) -> list[int]:
|
||||
"""规范形里的位置 → literals 里的位置（template_of 要的）；对不上的不要。"""
|
||||
found = (self.positions[i] for i in canonical_positions if i < len(self.positions))
|
||||
return sorted(p for p in found if p is not None)
|
||||
|
||||
|
||||
def shape_of(snap: Any, kinds: dict[str, str | None]) -> Shape | None:
|
||||
from app.data.sql_semantics import normalize_sql
|
||||
from app.data.sql_semantics import normalize_sql, sql_shape
|
||||
|
||||
if not isinstance(snap, dict) or not isinstance(snap.get("sql"), str) or not snap["sql"].strip():
|
||||
return None
|
||||
source = str(snap.get("source") or "")
|
||||
norm = normalize_sql(snap["sql"], kinds.get(source))
|
||||
return Shape(source, norm.text, norm.literals, norm.parse_error)
|
||||
found = sql_shape(snap["sql"], kinds.get(source)) if not norm.parse_error else None
|
||||
if found is None:
|
||||
return Shape(source, norm.text, norm.literals, norm.parse_error, key=f"text:{norm.text}",
|
||||
canonical=norm.literals, positions=tuple(range(len(norm.literals))))
|
||||
return Shape(source, norm.text, norm.literals, None, key=f"shape:{found.key}", canonical=found.literals,
|
||||
positions=found.positions)
|
||||
|
||||
|
||||
def varying(literals: Sequence[Sequence[str]]) -> list[int]:
|
||||
@@ -174,11 +196,13 @@ def template_of(shape: Shape, params: Sequence[int], dialect: str | None, *,
|
||||
class _Member:
|
||||
run_id: str
|
||||
artifact: str
|
||||
#: 按规范形里的先后排的字面量（Shape.canonical）
|
||||
literals: tuple[str, ...]
|
||||
conversation_id: str | None
|
||||
turn_id: str | None
|
||||
question: str
|
||||
related: list[dict[str, Any]] = field(default_factory=list)
|
||||
shape: Shape | None = None
|
||||
|
||||
|
||||
def referenced_queries(content: dict[str, Any]) -> list[str]:
|
||||
@@ -220,6 +244,28 @@ async def _dismissed(session: Any) -> set[str]:
|
||||
AuditEvent.action == DISMISS_ACTION, AuditEvent.subject_kind == SUBJECT_KIND))).all() if sid}
|
||||
|
||||
|
||||
async def _dismissed_members(session: Any) -> list[set[str]]:
|
||||
"""问题聚类上点过「不是口径」时记下的成员（每条一组稳定 id）。老记录没记成员、记的读不懂（不是字符串列表）的不算。"""
|
||||
from app.db.models import AuditEvent
|
||||
|
||||
out: list[set[str]] = []
|
||||
for (data,) in (await session.execute(select(AuditEvent.data).where(
|
||||
AuditEvent.action == DISMISS_ACTION, AuditEvent.subject_kind == SUBJECT_KIND))).all():
|
||||
members = data.get("members") if isinstance(data, dict) else None
|
||||
if isinstance(members, list) and members and all(isinstance(m, str) and m for m in members):
|
||||
out.append(set(members))
|
||||
return out
|
||||
|
||||
|
||||
def _cluster_dismissed(cluster: Any, dismissed: set[str], members: list[set[str]]) -> bool:
|
||||
"""这一类点过「不是口径」没有：类的 id 记过；以前按种子问题原文算的 id 记过；或者过半的成员是某一次记下的。"""
|
||||
legacy = item_id("cluster", " ".join(cluster.members[0].text.split()))
|
||||
if f"cluster:{cluster.id}" in dismissed or legacy in dismissed:
|
||||
return True
|
||||
keys = set(cluster.keys)
|
||||
return any(len(keys & seen) * 2 > len(keys) for seen in members)
|
||||
|
||||
|
||||
async def _frozen(session: Any) -> dict[str, list[dict[str, Any]]]:
|
||||
"""来源运行 → 从它固化出的工作流 [{workflow_id, name, status, published_version}]（origin.sources）。"""
|
||||
from app.db.models import Workflow
|
||||
@@ -258,6 +304,7 @@ async def queue(session: Any, *, principal: Any) -> dict[str, Any]:
|
||||
from app.ledger import store
|
||||
|
||||
dismissed = await _dismissed(session)
|
||||
dismissed_members = await _dismissed_members(session)
|
||||
frozen = await _frozen(session)
|
||||
kinds = {name: kind for name, kind in (await session.execute(select(DataSource.name, DataSource.kind))).all()}
|
||||
runs = list((await session.execute(select(Run).where(
|
||||
@@ -297,31 +344,38 @@ async def queue(session: Any, *, principal: Any) -> dict[str, Any]:
|
||||
info = turns.get(run_id) or {}
|
||||
related = [{k: c.get(k) for k in ("caliber", "workflow_id", "version", "level")}
|
||||
for c in row.get("candidates") or [] if c.get("status") == "related"]
|
||||
groups.setdefault((shape.source, shape.text), []).append(_Member(
|
||||
run_id, query, shape.literals, info.get("conversation_id"), info.get("turn_id"),
|
||||
str(info.get("question") or ""), related))
|
||||
groups.setdefault((shape.source, shape.key), []).append(_Member(
|
||||
run_id, query, shape.canonical, info.get("conversation_id"), info.get("turn_id"),
|
||||
str(info.get("question") or ""), related, shape))
|
||||
return groups
|
||||
|
||||
groups = await asyncio.to_thread(read)
|
||||
items: list[dict[str, Any]] = []
|
||||
for (source, text), members in groups.items():
|
||||
for (source, key), members in groups.items():
|
||||
once = list({m.run_id: m for m in members}.values())
|
||||
conversations = {m.conversation_id or f"run:{m.run_id}" for m in once}
|
||||
if len(once) < REPEAT_RUNS or len(conversations) < REPEAT_CONVERSATIONS:
|
||||
continue
|
||||
iid = item_id("shape", f"{source}\n{text}")
|
||||
if iid in dismissed:
|
||||
iid = item_id("shape", f"{source}\n{key}")
|
||||
# 以前按规范文本记的「不是口径」：组里任何一条的规范文本对得上就照样不进队
|
||||
legacy = {item_id("shape", f"{source}\n{m.shape.text}") for m in once if m.shape is not None}
|
||||
if iid in dismissed or legacy & dismissed:
|
||||
continue
|
||||
params = varying([m.literals for m in once])
|
||||
related = list({r["caliber"]: r for m in once for r in m.related if r.get("caliber")}.values())
|
||||
rep = once[0]
|
||||
shape = rep.shape
|
||||
if shape is None:
|
||||
continue
|
||||
varied = varying([m.literals for m in once])
|
||||
params = [(i, p) for i in varied if i < len(shape.positions) and (p := shape.positions[i]) is not None]
|
||||
related = list({r["caliber"]: r for m in once for r in m.related if r.get("caliber")}.values())
|
||||
items.append({
|
||||
"id": iid, "kind": "shape", "source": source, "shape": text, "runs": len(once),
|
||||
"id": iid, "kind": "shape", "source": source, "shape": shape.text, "runs": len(once),
|
||||
"conversations": len(conversations), "question": rep.question,
|
||||
"turns": [_turn_view(m) for m in once[:TURNS_LIMIT]],
|
||||
"params": [{"index": i, "distinct": len({m.literals[i] for m in once if len(m.literals) > i})}
|
||||
for i in params],
|
||||
"constants": len(rep.literals) - len(params), "related": related,
|
||||
# index 是代表那条规范文本里的第几个「?」；distinct 按规范形对齐了数
|
||||
"params": [{"index": p, "distinct": len({m.literals[i] for m in once if len(m.literals) > i})}
|
||||
for i, p in sorted(params, key=lambda ip: ip[1])],
|
||||
"constants": len(shape.literals) - len(params), "related": related,
|
||||
"representative": {"run_id": rep.run_id, "query": rep.artifact},
|
||||
"peers": [m.run_id for m in once[1:]],
|
||||
"frozen_into": _frozen_of([m.run_id for m in once], frozen),
|
||||
@@ -332,8 +386,8 @@ async def queue(session: Any, *, principal: Any) -> dict[str, Any]:
|
||||
if cluster.count < REPEAT_RUNS:
|
||||
continue
|
||||
seed = cluster.members[0]
|
||||
iid = item_id("cluster", " ".join(seed.text.split()))
|
||||
if iid in dismissed:
|
||||
iid = f"cluster:{cluster.id}"
|
||||
if _cluster_dismissed(cluster, dismissed, dismissed_members):
|
||||
continue
|
||||
items.append({"id": iid, "kind": "cluster", "question": seed.text, "count": cluster.count,
|
||||
"turns": [{"run_id": q.run_id, "turn_id": q.turn_id, "question": q.text}
|
||||
@@ -387,12 +441,25 @@ async def _reused(session: Any, runs: list[Run], turns: dict[str, dict[str, Any]
|
||||
return out
|
||||
|
||||
|
||||
async def dismiss(session: Any, iid: str, *, kind: str | None = None, note: str | None = None) -> None:
|
||||
"""「不是口径」：写一条审计，之后这一项不再进队。"""
|
||||
async def dismiss(session: Any, iid: str, *, kind: str | None = None, note: str | None = None,
|
||||
principal: Any = None) -> None:
|
||||
"""「不是口径」：写一条审计，之后这一项不再进队。问题聚类另记下这一类现在的成员（服务端按同一批问题重新聚一次找到
|
||||
这一类，不收客户端传的成员；找不到——这会儿问题又多了、聚出来的已经不是这一类——就只记 id，照实）。"""
|
||||
from app.auth.audit import record
|
||||
|
||||
members: list[str] = []
|
||||
if principal is not None and (kind == "cluster" or iid.startswith("cluster:")):
|
||||
from app.chat.questions import cluster_questions, recent_questions
|
||||
|
||||
try:
|
||||
questions = await recent_questions(session, principal, limit=QUESTION_LIMIT)
|
||||
found = next((c for c in await cluster_questions(questions) if f"cluster:{c.id}" == iid), None) \
|
||||
if questions else None
|
||||
members = found.keys if found is not None else []
|
||||
except Exception: # noqa: BLE001 - 取向量出错：只记 id，「不是口径」照样记上
|
||||
logger.exception("「不是口径」时重新聚类出错，只按 id 记")
|
||||
await record(session, DISMISS_ACTION, SUBJECT_KIND, iid,
|
||||
data={k: v for k, v in {"kind": kind, "note": note}.items() if v})
|
||||
data={k: v for k, v in {"kind": kind, "note": note, "members": members}.items() if v})
|
||||
await session.commit()
|
||||
|
||||
|
||||
@@ -447,7 +514,7 @@ async def freeze(session: Any, run_id: str, *, peers: Sequence[str] = (), name:
|
||||
raise CertifyRefused(409, NO_QUERIES)
|
||||
kinds = {n: k for n, k in (await session.execute(select(DataSource.name, DataSource.kind))).all()}
|
||||
|
||||
# 同组的别的运行：它们封存的查询里同形状的那一条的字面量
|
||||
# 同组的别的运行：它们封存的查询里同形状（规范形相同，见 Shape.key）的那一条的字面量，按规范形里的先后
|
||||
peer_literals: dict[tuple[str, str], list[tuple[str, ...]]] = {}
|
||||
peer_sources: list[dict[str, Any]] = []
|
||||
peer_runs = [r for r in [await session.get(Run, p) for p in dict.fromkeys(peers) if p != run_id] if r is not None]
|
||||
@@ -460,7 +527,7 @@ async def freeze(session: Any, run_id: str, *, peers: Sequence[str] = (), name:
|
||||
except ValueError:
|
||||
continue
|
||||
if shape is not None and not shape.problem:
|
||||
peer_literals.setdefault((shape.source, shape.text), []).append(shape.literals)
|
||||
peer_literals.setdefault((shape.source, shape.key), []).append(shape.canonical)
|
||||
sealed = await _sealed(run_id)
|
||||
shapes: list[Shape] = []
|
||||
snaps: list[dict[str, Any]] = []
|
||||
@@ -485,7 +552,7 @@ async def freeze(session: Any, run_id: str, *, peers: Sequence[str] = (), name:
|
||||
metrics: list[dict[str, Any]] = []
|
||||
multi = False
|
||||
for i, (shape, snap) in enumerate(zip(shapes, snaps)):
|
||||
params = varying([shape.literals, *peer_literals.get((shape.source, shape.text), [])])
|
||||
params = shape.params(varying([shape.canonical, *peer_literals.get((shape.source, shape.key), [])]))
|
||||
template, new_fields = template_of(shape, params, kinds.get(shape.source), start=len(fields) + 1)
|
||||
fields += new_fields
|
||||
nodes.append({"id": nid(i), "type": "tool", "position": {"x": 0, "y": 0},
|
||||
|
||||
@@ -47,8 +47,11 @@ _NUMBER = re.compile(r"^\s*-?\d+(?:\.\d+)?\s*$")
|
||||
#: 写给模型的拒收说明（也上界面：过程里那一行）
|
||||
NOT_LISTED = "只能选用这一轮查看到的已发布口径：「{workflow}」v{version} 不在里面（口径要先 look，版本以查看结果为准）"
|
||||
WRONG_VERSION = "「{workflow}」这一轮能选用的是 v{latest}（最新的已发布版本），不是 v{version}"
|
||||
CARD_UNKNOWN = "「{workflow}」v{version} 里没有口径卡「{card}」（有：{cards}）"
|
||||
CARD_AMBIGUOUS = "「{workflow}」v{version} 有几张口径卡（{cards}），请在 card 里写选用哪一张"
|
||||
#: card 写错时交回的原因要能一次改对：说清 card 填什么、可填哪几个（节点 id）；写成了自己起的 id 的另说一句
|
||||
CARD_UNKNOWN = ("「{workflow}」v{version} 里没有口径卡「{card}」：card 要填那一版工作流图里口径卡的节点 id，可填的有 {cards}"
|
||||
"（照抄查看结果里 use_caliber 写法中的 card）")
|
||||
CARD_IS_OP_ID = "；「{card}」是你给这次选用起的 id，不是口径卡的节点 id"
|
||||
CARD_AMBIGUOUS = "「{workflow}」v{version} 有几张口径卡（节点 id：{cards}），请在 card 里写选用哪一张的节点 id"
|
||||
NOT_COMPOSABLE = "「{workflow}」v{version} 的口径卡「{card}」不能组合：{reason}"
|
||||
BAD_ID = "use_caliber 的 id 要写成简短的英文小写加下划线（例如 c1），它就是口径卡的节点 id，当前为「{id}」"
|
||||
ID_TAKEN = "use_caliber 的 id「{id}」和图里已有的节点「{node}」撞了，请换一个"
|
||||
@@ -233,12 +236,14 @@ class Composer:
|
||||
if str(version) != str(latest):
|
||||
raise ComposeRefused(WRONG_VERSION.format(workflow=name, latest=latest, version=version))
|
||||
card = str(op.get("card") or "").strip()
|
||||
listed = [f"{o.card}（{o.node_id}）" for o in cards]
|
||||
listed = [f"「{o.node_id}」（口径卡「{o.card}」）" for o in cards]
|
||||
if card:
|
||||
hit = next((o for o in cards if card in (o.node_id, o.card)), None)
|
||||
if hit is None:
|
||||
raise ComposeRefused(CARD_UNKNOWN.format(workflow=name, version=latest, card=card,
|
||||
cards=_names(listed)))
|
||||
why = CARD_UNKNOWN.format(workflow=name, version=latest, card=card, cards=_names(listed))
|
||||
if card == str(op.get("id") or "").strip():
|
||||
why += CARD_IS_OP_ID.format(card=card)
|
||||
raise ComposeRefused(why)
|
||||
return hit
|
||||
if len(cards) > 1:
|
||||
raise ComposeRefused(CARD_AMBIGUOUS.format(workflow=name, version=latest, cards=_names(listed)))
|
||||
|
||||
@@ -162,6 +162,78 @@ def card_expressions(cards: list[Mapping[str, Any]]) -> list[tuple[str, str, str
|
||||
return out
|
||||
|
||||
|
||||
def card_reads(cards: list[Mapping[str, Any]]) -> list[tuple[str, str, tuple[tuple[str, str], ...]]]:
|
||||
"""口径版本里每张源头口径卡的每个指标读了哪几格的列：(卡的节点 id, 指标 id, ((取数节点, 列), …))，按式子里出现的先后、
|
||||
去重。只认 cell(nodes.X, 行, '列') 这种输入（同 card_expressions）；有别的写法的指标不收。比对时口径一侧按它投影到
|
||||
「这个指标引用的列」（ledger/match 的 _projected）。"""
|
||||
import ast
|
||||
|
||||
from app.engine.expressions import ExpressionError, cell_parts, leaf_refs, parse_expression
|
||||
|
||||
out: list[tuple[str, str, tuple[tuple[str, str], ...]]] = []
|
||||
for card in cards:
|
||||
for metric in card.get("metrics") or []:
|
||||
try:
|
||||
tree = parse_expression(str((metric or {}).get("expression") or ""))[0]
|
||||
except ExpressionError:
|
||||
continue
|
||||
reads: list[tuple[str, str]] = []
|
||||
for leaf in leaf_refs(tree):
|
||||
parts = cell_parts(leaf)
|
||||
if parts is None or not parts[0].startswith("nodes."):
|
||||
break
|
||||
try:
|
||||
column = ast.literal_eval(parts[2])
|
||||
except (ValueError, SyntaxError):
|
||||
break
|
||||
if not isinstance(column, str):
|
||||
break
|
||||
read = (parts[0][len("nodes."):], column)
|
||||
if read not in reads:
|
||||
reads.append(read)
|
||||
else:
|
||||
if reads:
|
||||
out.append((str(card.get("node_id")), str(metric.get("id")), tuple(reads)))
|
||||
return out
|
||||
|
||||
|
||||
def referenced_columns(docs: list[Mapping[str, Any]],
|
||||
load_card: Callable[[str], Mapping[str, Any] | None]) -> dict[str, set[str]]:
|
||||
"""报告里引用了哪条查询的哪几列：{查询快照: {列名}}。和 label_doc 标的是同一批数——单元格（[[v:]]、[[table:]] 里的格）
|
||||
按证据目录找到的快照和列；指标（[[m:]]）按指标集里这个指标的输入（落在查询快照一格上的）。比对时按它投影：只比被
|
||||
引用的那几列（ledger/match 的 _projected）。"""
|
||||
from app.engine.evidence import iter_units
|
||||
|
||||
out: dict[str, set[str]] = {}
|
||||
cards: dict[str, Mapping[str, Any] | None] = {}
|
||||
for doc in docs:
|
||||
catalog = doc.get("catalog") if isinstance(doc.get("catalog"), dict) else {}
|
||||
for _, unit in iter_units(doc):
|
||||
for seg in unit.get("segments") or []:
|
||||
if not isinstance(seg, dict) or seg.get("kind") not in ("number", "value") or not seg.get("id"):
|
||||
continue
|
||||
cite = seg.get("cite") if isinstance(seg.get("cite"), dict) else {}
|
||||
if cite.get("status") != "resolved":
|
||||
continue
|
||||
entry = catalog.get(cite.get("alias")) if isinstance(catalog.get(cite.get("alias")), dict) else {}
|
||||
locator = cite.get("locator") if isinstance(cite.get("locator"), dict) else {}
|
||||
artifact = str(entry.get("artifact") or "")
|
||||
if cite.get("kind") == "cell" and artifact and isinstance(locator.get("column"), str):
|
||||
out.setdefault(artifact, set()).add(locator["column"])
|
||||
elif cite.get("kind") == "metric" and artifact:
|
||||
if artifact not in cards:
|
||||
cards[artifact] = load_card(artifact)
|
||||
card = cards[artifact] or {}
|
||||
metric = next((m for m in card.get("metrics") or []
|
||||
if isinstance(m, dict) and m.get("id") == locator.get("metric")), None)
|
||||
for item in (metric or {}).get("inputs") or []:
|
||||
where = item.get("locator") if isinstance(item, dict) else None
|
||||
if isinstance(item, dict) and item.get("artifact") and item.get("via") in _SNAPSHOT_VIAS \
|
||||
and isinstance(where, dict) and isinstance(where.get("column"), str):
|
||||
out.setdefault(str(item["artifact"]), set()).add(where["column"])
|
||||
return out
|
||||
|
||||
|
||||
def metric_label(card: Mapping[str, Any] | None, metric_id: Any, queries: Mapping[str, Mapping[str, Any]],
|
||||
calibers: Mapping[str, Mapping[str, Any]]) -> dict[str, Any]:
|
||||
"""指标的标签（见模块说明）。card 是封存的指标集工件（读不出为 None）；calibers 是 {「工作流@版本」: {level, cards}}，
|
||||
@@ -359,5 +431,5 @@ def related_view(item: Mapping[str, Any], *, hide: Callable[[Any], bool], shown:
|
||||
|
||||
|
||||
__all__ = ["CERTIFIED_NOTE", "HOWS", "KINDS", "LEVEL_TEXT", "POST_SEAL_NOTE", "RETRO_NOTE", "binding_view", "caliber_key",
|
||||
"card_expressions", "cell_label", "diff_view", "label_doc", "metric_label", "related_view", "view_label",
|
||||
"whole_of"]
|
||||
"card_expressions", "card_reads", "cell_label", "diff_view", "label_doc", "metric_label",
|
||||
"referenced_columns", "related_view", "view_label", "whole_of"]
|
||||
|
||||
+173
-30
@@ -11,6 +11,20 @@ MAX_CANDIDATES 份（按工作流、版本、节点排好的先后）。
|
||||
|
||||
**怎么比。** data/sql_semantics.match_template：先绑定参数（只是启发），再按绑定渲染口径的取数 SQL 交 explain_sql_diff 证明。
|
||||
|
||||
**按引用的列投影**（_projected，algo 3）。助手写的 SQL 总会多选几列（订单数、日期常量、MIN / MAX……），口径的取数也只选
|
||||
它的指标要的那几列：整条比永远不等价。整条比不等价时，答案一侧按报告引用了这条查询的哪几列（labels.referenced_columns：
|
||||
单元格的列、指标输入落在的列）投影，口径一侧按口径卡上每个指标读了这个取数节点的哪几列（labels.card_reads）投影
|
||||
（sql_semantics.project_select / project_template：去掉没被引用、去掉也不改变被引用那几列取值的选择项，认不准的一律
|
||||
不投影），投影后的两条照样交 match_template 绑定、判等价——不另开一条给信任的路，判等价那一步一点不放宽。
|
||||
|
||||
被引用的列两边怎么对上：答案一侧按列名在查询快照记的结果列（columns）里找位置，结果列按位置对上选择列表的各项（列数
|
||||
和选择列表的项数要一样）；口径一侧按口径卡 cell(nodes.取数, 行, '列') 里写的列名，在取数 SQL 选择列表的输出名（别名，
|
||||
没有别名的是列名，不分大小写，同名的不止一项时不投影）里找位置。投影后两边留下的几列按位置配对（和整条比时的列映射一样：
|
||||
外层别名不计、列的先后计），两边列数一样才比；答案一侧留下两三列时另试调换先后。每份口径取数最多试 MAX_PROJECTIONS 种。
|
||||
证明了的候选记 projection {ours, dropped, caliber, caliber_dropped}；没证明的给第一种投影的比较（差在哪是投影之后的说法，
|
||||
答案多选了列的在最前面写一句「这次多查了 N 列，不影响这个数」）。没有投影可试的（报告没引用这条查询、两边都不投影、
|
||||
投影不了）照旧是整条比的结果，逐对一字不差。
|
||||
|
||||
**助手选用的（composed）。** 问数据的助手 use_caliber 选用口径时，取数节点是服务端照那一版拷进来的（ledger/compose.py），
|
||||
口径卡的 caliber_from 带着改名映射 {原 id: 新 id}。这几个节点交回的查询照样要证明：证明了的那一份记 how=composed，查询
|
||||
状态记 composed——数确实来自口径，但「这份口径答的就是这个问题」是助手凭问法判断的，不能说成「等价已证明」（拍板 L1：
|
||||
@@ -40,6 +54,7 @@ import logging
|
||||
import time
|
||||
import weakref
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from collections.abc import Collection, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
@@ -62,6 +77,8 @@ CONCURRENCY = 2
|
||||
GRACE_S = 2.0
|
||||
#: 每条查询最多比几份口径
|
||||
MAX_CANDIDATES = 8
|
||||
#: 按引用的列投影之后，每份口径取数最多再试几种比法（答案一侧留下的列的先后 × 口径一侧的列组，见 _projected）
|
||||
MAX_PROJECTIONS = 12
|
||||
#: 启动时补比最近几天、最多几个
|
||||
BACKFILL_DAYS = 7
|
||||
BACKFILL_LIMIT = 200
|
||||
@@ -76,6 +93,8 @@ PARSE_FAILED = "这次的 SQL 无法解析，没有比对"
|
||||
MERGED = "合并查询的格不和口径比对（它的数来自各个输入的查询）"
|
||||
COMPOSED_UNPROVEN = "这条查询出自助手选用的口径，但和口径的取数没能证明一致（可能被改过）"
|
||||
MORE_CANDIDATES = "另有 {n} 份相关口径没有比对（每条查询最多比 {max} 份）"
|
||||
#: 按引用的列投影之后没证明等价时，差在哪的第一句：答案只是多选了几列（去掉它们不影响被引用的数，见 sql_semantics 的投影）
|
||||
EXTRA_COLUMNS = "这次多查了 {n} 列，不影响这个数"
|
||||
|
||||
#: 不比对的原因（GET 账本时照实说）
|
||||
NOT_FOUND = "运行记录不存在"
|
||||
@@ -124,7 +143,8 @@ class Fetch:
|
||||
@dataclass(frozen=True)
|
||||
class Caliber:
|
||||
"""一个工作流最新的已发布、未退役版本（只收里面有查库节点的）。fields 是入口节点的字段原样（取缺省值用，不进工件），
|
||||
cards 是源头口径卡的节点 id，expressions 是这些卡上每个指标换成记号后的式子（labels.card_expressions）。"""
|
||||
cards 是源头口径卡的节点 id，expressions 是这些卡上每个指标换成记号后的式子（labels.card_expressions），reads 是每个
|
||||
指标读了哪几个取数节点的哪几列（labels.card_reads：口径一侧按它投影，不进工件）。"""
|
||||
|
||||
workflow_id: str
|
||||
version: int
|
||||
@@ -134,6 +154,7 @@ class Caliber:
|
||||
fields: tuple[dict[str, Any], ...] = ()
|
||||
cards: tuple[str, ...] = ()
|
||||
expressions: tuple[tuple[str, str, str], ...] = ()
|
||||
reads: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...] = ()
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
@@ -201,7 +222,8 @@ def caliber_of(workflow_id: str, row: Any) -> Caliber | None:
|
||||
return Caliber(workflow_id=workflow_id, version=int(row.version), level=str(getattr(row, "level", None) or ""),
|
||||
graph_hash=getattr(row, "graph_hash", None), fetches=tuple(fetches), fields=tuple(fields),
|
||||
cards=tuple(c["node_id"] for c in cards),
|
||||
expressions=tuple(ledger_labels.card_expressions(cards)))
|
||||
expressions=tuple(ledger_labels.card_expressions(cards)),
|
||||
reads=tuple(ledger_labels.card_reads(cards)))
|
||||
|
||||
|
||||
async def load_calibers(session: Any) -> list[Caliber]:
|
||||
@@ -331,12 +353,108 @@ def composed_nodes(graph: Any) -> dict[str, tuple[str, str]]:
|
||||
return out
|
||||
|
||||
|
||||
def _wanted(query: Query, referenced: Mapping[str, Collection[str]] | None) -> tuple[int, ...] | None:
|
||||
"""报告引用了这条查询的哪几列：按列名在查询快照记的结果列里找位置（按位置先后，同名的几列都算）。没引用、快照没记
|
||||
结果列的为 None（不投影）。"""
|
||||
names = (referenced or {}).get(query.artifact)
|
||||
if not names or query.columns is None:
|
||||
return None
|
||||
return tuple(i for i, c in enumerate(query.columns) if c in names) or None
|
||||
|
||||
|
||||
def _caliber_sets(caliber: Caliber, fetch: Fetch, names: list[str]) -> list[tuple[int, ...]]:
|
||||
"""口径一侧按哪几组列投影：口径卡上每个指标读了这个取数节点的哪几列（labels.card_reads），按列名（不分大小写）在取数
|
||||
SQL 的输出名里找位置、按位置排好；最后加上所有指标读到的列合在一起的那一组。找不到、同名的不止一项的那一组不要。"""
|
||||
lowered = [n.lower() for n in names]
|
||||
out: list[tuple[int, ...]] = []
|
||||
union: set[int] = set()
|
||||
for _, _, reads in caliber.reads:
|
||||
columns = [column for node, column in reads if node == fetch.node_id]
|
||||
if not columns:
|
||||
continue
|
||||
found = [[i for i, n in enumerate(lowered) if n == c.lower()] for c in columns]
|
||||
if any(len(f) != 1 for f in found):
|
||||
continue
|
||||
group = tuple(sorted({f[0] for f in found}))
|
||||
union |= set(group)
|
||||
if group not in out:
|
||||
out.append(group)
|
||||
whole = tuple(sorted(union))
|
||||
if whole and whole not in out:
|
||||
out.append(whole)
|
||||
return out
|
||||
|
||||
|
||||
def _projection_view(query: Query, ours: Any, theirs: Any) -> dict[str, Any]:
|
||||
"""候选上记的「按引用的列比的」：答案留下的列（快照里的列名）、去掉了几列，口径留下的列、去掉了几列。"""
|
||||
return {"ours": [str((query.columns or [])[i]) for i in ours.kept], "dropped": ours.dropped,
|
||||
"caliber": list(theirs.names), "caliber_dropped": theirs.dropped}
|
||||
|
||||
|
||||
def _projected(caliber: Caliber, fetch: Fetch, query: Query, wanted: tuple[int, ...],
|
||||
deadline: float | None) -> dict[str, Any] | None:
|
||||
"""整条比不等价时按引用的列投影再比（见模块说明）：证明了的候选（带 projection）；都没证明时第一种投影的那次比较；
|
||||
没有投影可试的为 None（照旧用整条比的结果）。"""
|
||||
from itertools import permutations
|
||||
|
||||
from app.data.sql_semantics import FacetDiff, match_template, output_names, project_select, project_template
|
||||
|
||||
width = len(query.columns or [])
|
||||
whole = project_select(query.sql, tuple(range(width)), query.dialect, width=width)
|
||||
if whole.sql is None:
|
||||
return None
|
||||
ours = []
|
||||
if len(wanted) < width:
|
||||
cut = project_select(query.sql, wanted, query.dialect, width=width)
|
||||
if cut.sql is not None:
|
||||
ours.append(cut)
|
||||
ours = ours or [whole]
|
||||
for base in list(ours):
|
||||
if 2 <= len(base.kept) <= 3:
|
||||
for order in permutations(base.kept):
|
||||
if order != base.kept and (p := project_select(query.sql, order, query.dialect, width=width)).sql:
|
||||
ours.append(p)
|
||||
names = output_names(fetch.sql, query.dialect, template=True)
|
||||
if names is None:
|
||||
return None
|
||||
theirs = [p for group in _caliber_sets(caliber, fetch, names) if len(group) < len(names)
|
||||
for p in [project_template(fetch.sql, group, query.dialect)] if p.sql is not None]
|
||||
theirs.append(project_template(fetch.sql, tuple(range(len(names))), query.dialect))
|
||||
first: tuple[Any, Any, Any] | None = None
|
||||
tries = 0
|
||||
for a in ours:
|
||||
for c in theirs:
|
||||
if c.sql is None or len(a.kept) != len(c.kept) or (not a.dropped and not c.dropped
|
||||
and a.kept == whole.kept):
|
||||
continue # 两边都没投影：就是整条比过的那一次
|
||||
if tries >= MAX_PROJECTIONS or (deadline is not None and time.monotonic() > deadline):
|
||||
break
|
||||
tries += 1
|
||||
m = match_template(c.sql, list(caliber.fields), a.sql, query.dialect, schema=query.schema,
|
||||
names=query.names, columns=[str((query.columns or [])[i]) for i in a.kept],
|
||||
deadline=deadline)
|
||||
if m.equivalent:
|
||||
return {**_candidate(caliber, fetch, m), "projection": _projection_view(query, a, c)}
|
||||
if first is None:
|
||||
first = (a, c, m)
|
||||
if first is None:
|
||||
return None
|
||||
a, c, m = first
|
||||
cand = {**_candidate(caliber, fetch, m), "projection": _projection_view(query, a, c)}
|
||||
if a.dropped and isinstance(cand.get("diff"), dict):
|
||||
note = FacetDiff("projection", None, None, EXTRA_COLUMNS.format(n=a.dropped)).as_dict()
|
||||
cand["diff"] = {**cand["diff"], "items": [note, *cand["diff"]["items"]]}
|
||||
return cand
|
||||
|
||||
|
||||
def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline: float | None = None,
|
||||
composed: dict[str, tuple[str, str]] | None = None) -> list[dict[str, Any]]:
|
||||
composed: dict[str, tuple[str, str]] | None = None,
|
||||
referenced: Mapping[str, Collection[str]] | None = None) -> list[dict[str, Any]]:
|
||||
"""每条查询和预筛出来的口径取数逐个比（match_template），给出 [{artifact, node_id, source, tables, status, reason?,
|
||||
candidates}]。status：proven（和某一份等价）/ composed（助手选用的那一份，证明了）/ related（有同源同表的口径，都不
|
||||
等价）/ none（没有相关的口径）/ skipped（没比完：超时、解析失败、快照读不出）。composed 是 composed_nodes 的结果，
|
||||
不给就没有助手选用的。纯计算，从不抛异常。评测逐题比 SQL 也调它。"""
|
||||
不给就没有助手选用的。referenced 是报告引用了哪条查询的哪几列（labels.referenced_columns：{查询快照: {列名}}）：
|
||||
给了的，整条比不等价时按引用的列投影再比（_projected）；不给就只整条比。纯计算，从不抛异常。评测逐题比 SQL 也调它。"""
|
||||
from app.data.sql_semantics import match_template, normalize_sql
|
||||
|
||||
out: list[dict[str, Any]] = []
|
||||
@@ -347,6 +465,7 @@ def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline:
|
||||
out.append({**row, "status": "skipped", "reason": query.problem, "candidates": []})
|
||||
continue
|
||||
chosen = (composed or {}).get(str(query.node_id or ""))
|
||||
wanted = _wanted(query, referenced)
|
||||
pool = [(c, f) for c in calibers for f in c.fetches if f.source == query.source and f.tables & query.tables]
|
||||
if chosen is not None:
|
||||
# 助手选用的那一份排最前面：候选多的时候也一定比到它
|
||||
@@ -370,6 +489,11 @@ def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline:
|
||||
candidates.append(_skipped(caliber, fetch, "比对出错"))
|
||||
continue
|
||||
cand = _candidate(caliber, fetch, m)
|
||||
if not m.equivalent and m.comparable and wanted is not None:
|
||||
try:
|
||||
cand = _projected(caliber, fetch, query, wanted, deadline) or cand
|
||||
except Exception: # noqa: BLE001 - 投影再比是附加的一步：出错照旧用整条比的结果
|
||||
logger.exception("按引用的列投影再比出错（运行查询 %s，口径 %s）", query.artifact[:12], caliber.key)
|
||||
if chosen is not None and (caliber.key, fetch.node_id) == chosen:
|
||||
if cand["status"] == "proven":
|
||||
cand["how"] = "composed"
|
||||
@@ -389,27 +513,10 @@ def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline:
|
||||
return out
|
||||
|
||||
|
||||
def compute(sealed: Any, calibers: list[Caliber], kinds: dict[str, str | None], *, timeout: float) -> dict[str, Any]:
|
||||
"""一次运行的比对工件内容（见 ledger/store 的模块说明）。在线程里调：读快照、比 SQL、标数都是同步的活。"""
|
||||
from app.api.evidence import _choose, _reports
|
||||
def _card_loader(sealed: Any) -> Any:
|
||||
"""按工件读封存范围内的指标集（不在封存台账里的、读不出的为 None）。"""
|
||||
from app.core.artifact_store import load
|
||||
|
||||
deadline = time.monotonic() + timeout
|
||||
queries = read_queries(sealed, kinds)
|
||||
if not sealed.trusted:
|
||||
results = [{"artifact": q.artifact, "node_id": q.node_id, "source": q.source, "tables": sorted(q.tables),
|
||||
"status": "skipped", "reason": SEAL_BROKEN, "candidates": []} for q in queries]
|
||||
else:
|
||||
results = compare_queries(queries, calibers, deadline=deadline, composed=composed_nodes(sealed.graph))
|
||||
used = {c["caliber"] for q in results for c in q["candidates"]}
|
||||
considered = [c for c in calibers if c.key in used]
|
||||
by_key = {c.key: {"level": c.level, "fields": c.summary()["fields"], "expressions": list(c.expressions)}
|
||||
for c in considered}
|
||||
by_artifact = {q["artifact"]: q for q in results}
|
||||
# 合并查询的快照不比（read_queries 跳过了它），引用了它的格照实写为什么没比对；不进工件的 queries，统计不数它
|
||||
for artifact in sealed.queries:
|
||||
by_artifact.setdefault(artifact, {"artifact": artifact, "status": "skipped", "reason": MERGED,
|
||||
"candidates": []})
|
||||
sealed_cards = {artifact for _, artifact in sealed.ledger}
|
||||
|
||||
def load_card(artifact: str) -> dict[str, Any] | None:
|
||||
@@ -421,12 +528,47 @@ def compute(sealed: Any, calibers: list[Caliber], kinds: dict[str, str | None],
|
||||
return None
|
||||
return card if isinstance(card, dict) else None
|
||||
|
||||
return load_card
|
||||
|
||||
|
||||
def referenced_of(sealed: Any) -> dict[str, set[str]]:
|
||||
"""这次运行封存的报告引用了哪条查询的哪几列（labels.referenced_columns，只读哈希核对一致的报告）：compare_queries 的
|
||||
referenced。评测逐题比 SQL 时传它，就和账本一样按引用的列投影。在线程里调（读盘）。"""
|
||||
from app.api.evidence import _reports
|
||||
|
||||
docs = [r.doc for r in _reports(sealed) if r.doc is not None and r.hash_ok is True]
|
||||
return ledger_labels.referenced_columns(docs, _card_loader(sealed))
|
||||
|
||||
|
||||
def compute(sealed: Any, calibers: list[Caliber], kinds: dict[str, str | None], *, timeout: float) -> dict[str, Any]:
|
||||
"""一次运行的比对工件内容（见 ledger/store 的模块说明）。在线程里调：读快照、比 SQL、标数都是同步的活。"""
|
||||
from app.api.evidence import _choose, _reports
|
||||
|
||||
deadline = time.monotonic() + timeout
|
||||
queries = read_queries(sealed, kinds)
|
||||
load_card = _card_loader(sealed)
|
||||
found = _reports(sealed)
|
||||
readable = [report for report in found if report.doc is not None and report.hash_ok is True]
|
||||
if not sealed.trusted:
|
||||
results = [{"artifact": q.artifact, "node_id": q.node_id, "source": q.source, "tables": sorted(q.tables),
|
||||
"status": "skipped", "reason": SEAL_BROKEN, "candidates": []} for q in queries]
|
||||
else:
|
||||
referenced = ledger_labels.referenced_columns([report.doc for report in readable], load_card)
|
||||
results = compare_queries(queries, calibers, deadline=deadline, composed=composed_nodes(sealed.graph),
|
||||
referenced=referenced)
|
||||
used = {c["caliber"] for q in results for c in q["candidates"]}
|
||||
considered = [c for c in calibers if c.key in used]
|
||||
by_key = {c.key: {"level": c.level, "fields": c.summary()["fields"], "expressions": list(c.expressions)}
|
||||
for c in considered}
|
||||
by_artifact = {q["artifact"]: q for q in results}
|
||||
# 合并查询的快照不比（read_queries 跳过了它），引用了它的格照实写为什么没比对；不进工件的 queries，统计不数它
|
||||
for artifact in sealed.queries:
|
||||
by_artifact.setdefault(artifact, {"artifact": artifact, "status": "skipped", "reason": MERGED,
|
||||
"candidates": []})
|
||||
reports: dict[str, Any] = {}
|
||||
primary: str | None = None
|
||||
found = _reports(sealed)
|
||||
for report in found:
|
||||
if report.doc is not None and report.hash_ok is True:
|
||||
reports[report.node_id] = {"labels": ledger_labels.label_doc(report.doc, by_artifact, by_key, load_card)}
|
||||
for report in readable:
|
||||
reports[report.node_id] = {"labels": ledger_labels.label_doc(report.doc, by_artifact, by_key, load_card)}
|
||||
if found:
|
||||
primary = _choose(found, None, sealed).node_id
|
||||
whole = ledger_labels.whole_of((reports.get(primary) or {}).get("labels") or {}, by_key) if primary in reports \
|
||||
@@ -617,6 +759,7 @@ async def shutdown() -> None:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
|
||||
__all__ = ["BACKFILL_DAYS", "CONCURRENCY", "EXPLORE_ORIGINS", "TIMEOUT_S", "Caliber", "Fetch", "Query", "backfill",
|
||||
"caliber_of", "compare_queries", "composed_nodes", "compute", "drain", "ineligible", "load_calibers",
|
||||
"match_run", "pending", "read_queries", "schedule", "shutdown", "start_backfill", "wanted"]
|
||||
__all__ = ["BACKFILL_DAYS", "CONCURRENCY", "EXPLORE_ORIGINS", "EXTRA_COLUMNS", "MAX_PROJECTIONS", "TIMEOUT_S",
|
||||
"Caliber", "Fetch", "Query", "backfill", "caliber_of", "compare_queries", "composed_nodes", "compute",
|
||||
"drain", "ineligible", "load_calibers", "match_run", "pending", "read_queries", "referenced_of",
|
||||
"schedule", "shutdown", "start_backfill", "wanted"]
|
||||
|
||||
@@ -16,7 +16,8 @@
|
||||
的字段校验，ledger/compose.check_inputs 同一套），界面写「参数按这次的 SQL 推断，请核对」。
|
||||
|
||||
**列怎么配。** 只有一行结果时：口径卡上的指标式就是取数第一行某一列（cell(nodes.取数, 0, '列')）的，按那一列在口径取数
|
||||
结果里的位置配这次结果同一位置的列（等价时已实测列的先后计；不等价的两条 SQL 选择列表一样时位置照样对得上）。差异
|
||||
结果里的位置配这次结果同一位置的列（等价时已实测列的先后计；不等价的两条 SQL 选择列表一样时位置照样对得上）。比对时
|
||||
按引用的列投影过的（候选的 projection，ledger/match 的 _projected），按投影之后两边留下的列逐个配（这次多选的列不算）。差异
|
||||
有兜底说明（fallback：两条 SQL 对不上结构）、列数不同、一个都配不上时不猜，要人选（pairs：{ours: 这次的列, metric: 口径卡
|
||||
上的指标 id}），这个选择记进对账运行的 origin。
|
||||
|
||||
@@ -327,8 +328,9 @@ def _plain_cell(expression: str, fetch: str) -> str | None:
|
||||
|
||||
def plan_pairs(*, ours_columns: list[str], ours_rows: list[Any], hidden: set[str], caliber_columns: list[str] | None,
|
||||
metrics: list[dict[str, Any]], fetch: str, card: str, fallback: bool, chosen: Any,
|
||||
referenced: list[tuple[int, str]]) -> Plan:
|
||||
"""配列、（多行时）定行和键：交回 Plan，配不了的抛 ReconcileRefused（见模块说明）。纯计算。"""
|
||||
referenced: list[tuple[int, str]], projected: Any = None) -> Plan:
|
||||
"""配列、（多行时）定行和键：交回 Plan，配不了的抛 ReconcileRefused（见模块说明）。projected 是比对候选的 projection
|
||||
（按引用的列投影过的：{ours: [这次留下的列], caliber: [口径留下的列]}，按位置一一对应）。纯计算。"""
|
||||
def check_column(column: str) -> None:
|
||||
if column not in ours_columns:
|
||||
raise ReconcileRefused(422, PAIR_COLUMN.format(column=column))
|
||||
@@ -360,7 +362,17 @@ def plan_pairs(*, ours_columns: list[str], ours_rows: list[Any], hidden: set[str
|
||||
if fallback:
|
||||
raise ReconcileRefused(409, PAIRS_FALLBACK, NEEDS_PAIRS, extra=options)
|
||||
pairs = []
|
||||
if caliber_columns is not None and len(caliber_columns) == len(ours_columns):
|
||||
kept = _projected_pairs(projected, ours_columns)
|
||||
if kept is not None:
|
||||
# 按投影之后留下的列配：这次多选的列不算，口径一侧只看投影留下的那几列（列名不分大小写）
|
||||
for index, m in enumerate(metrics):
|
||||
column = _plain_cell(str(m.get("expression") or ""), fetch)
|
||||
ours = next((o for o, c in kept if column is not None and c.lower() == column.lower()), None)
|
||||
if ours is None or ours.lower() in hidden:
|
||||
continue
|
||||
pairs.append(Pair(ours=ours, metric=str(m.get("id")), index=index, name=str(m.get("name") or m.get("id")),
|
||||
unit=str(m.get("unit") or ""), decimals=m.get("decimals")))
|
||||
elif caliber_columns is not None and len(caliber_columns) == len(ours_columns):
|
||||
for index, m in enumerate(metrics):
|
||||
column = _plain_cell(str(m.get("expression") or ""), fetch)
|
||||
if column is None or column not in caliber_columns:
|
||||
@@ -375,6 +387,19 @@ def plan_pairs(*, ours_columns: list[str], ours_rows: list[Any], hidden: set[str
|
||||
return Plan(pairs=pairs)
|
||||
|
||||
|
||||
def _projected_pairs(projected: Any, ours_columns: list[str]) -> list[tuple[str, str]] | None:
|
||||
"""比对候选的 projection → [(这次的列, 口径的列)]，按位置一一对应；不是投影过的、两边列数不一样、这次的列不在结果里的
|
||||
为 None（照旧按整条的位置配）。"""
|
||||
if not isinstance(projected, dict):
|
||||
return None
|
||||
ours, theirs = projected.get("ours"), projected.get("caliber")
|
||||
if not isinstance(ours, list) or not isinstance(theirs, list) or not ours or len(ours) != len(theirs):
|
||||
return None
|
||||
if any(not isinstance(o, str) or o not in ours_columns or not isinstance(c, str) for o, c in zip(ours, theirs)):
|
||||
return None
|
||||
return list(zip(ours, theirs))
|
||||
|
||||
|
||||
def _plan_rows(ours_columns: list[str], rows: list[Any], hidden: set[str], caliber_columns: list[str] | None,
|
||||
referenced: list[tuple[int, str]], check_column: Any) -> Plan:
|
||||
if caliber_columns is None:
|
||||
@@ -536,7 +561,7 @@ async def reconcile(run_id: str, req: Request, *, principal: Any) -> Run:
|
||||
caliber_columns=_caliber_columns(template, inputs, list(pinned.entry_fields), kind, schema),
|
||||
metrics=pinned.metrics, fetch=str(fetch["id"]), card=pinned.card_title,
|
||||
fallback=bool((cand.get("diff") or {}).get("fallback")), chosen=req.pairs,
|
||||
referenced=_referenced(sealed, req.query))
|
||||
referenced=_referenced(sealed, req.query), projected=cand.get("projection"))
|
||||
graph = build(pinned, sql=snap["sql"], tool=f"{QUERY_PREFIX}{source}", plan=plan, inputs=inputs,
|
||||
fetch=str(fetch["id"]))
|
||||
|
||||
|
||||
@@ -8,8 +8,10 @@
|
||||
这时不再按来源和计入统计筛——调用方点名的就是要算的）。
|
||||
|
||||
- E1 等价命中：比对过的查询里 proven / composed / related / none / skipped 各几条（skipped 单独列，不混进 none），有候选口径
|
||||
的查询里各自的比率；证明等价的绑定是在哪一级配上的；没证明的原因（不可比、参数绑不上、配上了但不等价）；不等价的差异类型
|
||||
分布（time / filter / aggregate / join / other / parse，每份口径的差异每类只计一次）；逐个数的标签各几个、整题等价几次。
|
||||
的查询里各自的比率；证明等价的绑定是在哪一级配上的，其中几份是按引用的列投影之后才证明的（proven_projected，ledger/match
|
||||
的 _projected）；没证明的原因（不可比、参数绑不上、配上了但不等价）；不等价的差异类型分布（time / filter / aggregate /
|
||||
join / other / parse，每份口径的差异每类只计一次；投影之后那句「这次多查了 N 列」不是差异，不计）；逐个数的标签各几个、
|
||||
整题等价几次。
|
||||
- E2 对账：对账运行（origin=reconcile，第 3 波起由 ledger/reconcile 发起）发起、完成、失败各几次，对过账的答案几个，其中参数
|
||||
经人核对过的、对照的指标由人选的各几次；相关口径的差异说明里兜底说明的占比；对账结果上「说明有用 / 看不懂」的反馈
|
||||
（第 4 波，ledger/feedback：对账运行上的封存后事件 ledger.feedback，每人只算最后一次）各几个、有用的比率。
|
||||
@@ -42,6 +44,8 @@ REPEAT_CONVERSATIONS = 2
|
||||
ASPECT_GROUP = {"time": "time", "time_range": "time", "filter": "filter", "aggregate": "aggregate",
|
||||
"group_by": "aggregate", "join": "join", "tables": "join", "parse": "parse"}
|
||||
DIFF_GROUPS = ("time", "filter", "aggregate", "join", "other", "parse")
|
||||
#: 差在哪里不算差异的条目：投影之后的「这次多查了 N 列，不影响这个数」
|
||||
NOT_DIFFS = ("projection",)
|
||||
STATUSES = ("proven", "composed", "related", "none", "skipped")
|
||||
#: 聚类最多取几个问题（同治理接口的上限）
|
||||
QUESTION_LIMIT = 500
|
||||
@@ -98,6 +102,7 @@ def e1_of(contents: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
status = Counter({s: 0 for s in STATUSES})
|
||||
with_candidates = Counter({s: 0 for s in STATUSES})
|
||||
via: Counter[str] = Counter()
|
||||
projected = 0
|
||||
unproven: Counter[str] = Counter()
|
||||
diffs = Counter({g: 0 for g in DIFF_GROUPS})
|
||||
labels = Counter({k: 0 for k in ledger_labels.KINDS})
|
||||
@@ -113,6 +118,7 @@ def e1_of(contents: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
for cand in query.get("candidates") or []:
|
||||
if cand.get("status") == "proven":
|
||||
via[str(cand.get("via") or "parsed")] += 1
|
||||
projected += isinstance(cand.get("projection"), dict)
|
||||
continue
|
||||
if cand.get("status") == "skipped":
|
||||
continue
|
||||
@@ -123,7 +129,7 @@ def e1_of(contents: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
else:
|
||||
unproven["参数配上了，但和口径不等价"] += 1
|
||||
seen = {ASPECT_GROUP.get(str(item.get("aspect")), "other")
|
||||
for item in (cand.get("diff") or {}).get("items") or []}
|
||||
for item in (cand.get("diff") or {}).get("items") or [] if item.get("aspect") not in NOT_DIFFS}
|
||||
for group in seen:
|
||||
diffs[group] += 1
|
||||
for report in (content.get("reports") or {}).values():
|
||||
@@ -136,7 +142,8 @@ def e1_of(contents: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"runs": len(contents), "queries": queries, "status": dict(status),
|
||||
"with_candidates": candidates,
|
||||
"rates": {s: _ratio(with_candidates[s], candidates) for s in STATUSES if s != "none"},
|
||||
"proven_via": dict(via), "unproven_reasons": dict(unproven.most_common(20)), "diff_aspects": dict(diffs),
|
||||
"proven_via": dict(via), "proven_projected": projected, "unproven_reasons": dict(unproven.most_common(20)),
|
||||
"diff_aspects": dict(diffs),
|
||||
"labels": dict(labels), "whole_equivalent": whole,
|
||||
}
|
||||
|
||||
|
||||
@@ -31,7 +31,8 @@ FORMAT = "agentlab.caliber_match/1"
|
||||
#: 比对算法的版本：绑定、证明、标签的规则改了就加一，界面和统计据此分得清是哪一版算法比的。
|
||||
#: 2（第 3 波）：认出助手选用的口径（查询 composed、标签 how=composed），指标的标签记它取自哪几条查询（queries，认证队列
|
||||
#: 认「被报告引用的查询」用）
|
||||
ALGO = "2"
|
||||
#: 3：整条比不等价时按报告引用的列投影再比（ledger/match 的 _projected），候选记 projection；没投影的照旧
|
||||
ALGO = "3"
|
||||
#: 触发方式（见模块说明）。certified（第 3 波）：从问数据固化出的口径发布之后，对来源运行补比的（ledger/certify）
|
||||
TRIGGERS = ("sealed", "backfill", "on_demand", "certified")
|
||||
|
||||
|
||||
@@ -88,6 +88,9 @@ def _query_view(query: dict[str, Any], hider: _Hide, names: dict[str, str]) -> d
|
||||
item["how"] = cand["how"]
|
||||
if cand.get("columns") is not None:
|
||||
item["columns"] = cand["columns"]
|
||||
if isinstance(cand.get("projection"), dict):
|
||||
# 按引用的列投影之后比的（ledger/match 的 _projected）：两边各留下了哪几列、去掉了几列（列名，不带值）
|
||||
item["projection"] = cand["projection"]
|
||||
cands.append(item)
|
||||
out["candidates"] = cands
|
||||
return out
|
||||
|
||||
@@ -41,8 +41,8 @@ DETAIL_NOT_REUSABLE_PROMPT = " 不能复用：{reason}。请按表重新对"
|
||||
DETAIL_FIELDS_PROMPT = " 口径「{workflow}」v{version} 的入口字段：{fields}"
|
||||
DETAIL_FIELD_PROMPT = "{name}（{bits}）{desc}"
|
||||
DETAIL_NO_FIELDS_PROMPT = " 口径「{workflow}」v{version} 的入口没有字段"
|
||||
DETAIL_COMPOSE_PROMPT = (" 能组合：选用写法 {op}（id 自己起，它就是口径卡的节点 id；inputs 按上面的入口字段填，没写的取缺省值；"
|
||||
"metrics 写要用的指标 id）")
|
||||
DETAIL_COMPOSE_PROMPT = (" 能组合：选用写法 {op}（card 原样照抄，是那一版图里口径卡的节点 id；id 自己起，生成的口径卡用它当节点 id；"
|
||||
"inputs 按上面的入口字段填，没写的取缺省值；metrics 写要用的指标 id）")
|
||||
DETAIL_NOT_COMPOSABLE_PROMPT = " 不能组合：{reason}。这份口径只能作为参考，查库交给 agent"
|
||||
BRIEF_PROMPT = "· {name}（{workflow} v{version}）：字数超过上限，只列名字，要看指标式时点名再 look"
|
||||
MISSING_PROMPT = "没找到口径：{items}（只能查看已发布且未退役的版本里的口径卡；别拿名字相近的顶替）"
|
||||
|
||||
@@ -270,3 +270,155 @@ async def test_freezing_needs_a_matched_successful_run(client, monkeypatch):
|
||||
row = await S.explore(monkeypatch, S.ELSEWHERE, text="一共几家店，没有引用。")
|
||||
res = await client.post("/api/copilot/from-run", json={"run_id": row.id, "freeze": True})
|
||||
assert res.status_code == 409 and res.json()["detail"] == certify.NO_QUERIES
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 确定性重复按规范形分组：条件的先后不算，条件真不同的不算一组
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: 同一个形状的三种写法：条件换序、AND 两边换位、比较两边换位
|
||||
REORDERED = [
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '{s}' AND paid_at < '{e}'",
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE paid_at < '{e}' AND paid_at >= '{s}' AND status = 'paid'",
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE '{s}' <= paid_at AND ('paid' = status AND paid_at < '{e}')",
|
||||
]
|
||||
|
||||
|
||||
def test_conditions_in_another_order_are_the_same_shape_and_their_literals_line_up():
|
||||
from app.data.sql_semantics import sql_shape
|
||||
|
||||
shapes = [sql_shape(sql.format(s=s, e=e), "sqlite") for sql, (s, e) in zip(REORDERED, WEEKS)]
|
||||
assert all(sh is not None for sh in shapes)
|
||||
assert len({sh.key for sh in shapes}) == 1
|
||||
# 按规范形里的先后对齐：同一个位置是同一个条件里的那个字面量——变了的是两个日期，'paid' 在每一条里都落在同一个位置
|
||||
assert len(certify.varying([sh.literals for sh in shapes])) == 2
|
||||
assert len({sh.literals.index("'paid'") for sh in shapes}) == 1
|
||||
starts = {sh.literals.index(f"'{s}'") for sh, (s, _) in zip(shapes, WEEKS)}
|
||||
assert len(starts) == 1, "起始日期在每一条里都落在同一个位置"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("other", [
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND created_at >= '2026-09-01' AND paid_at < '2026-09-08'",
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' OR paid_at >= '2026-09-01' AND paid_at < '2026-09-08'",
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '2026-09-01'",
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at > '2026-09-01' AND paid_at < '2026-09-08'",
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE status = 1 AND paid_at >= '2026-09-01' AND paid_at < '2026-09-08'",
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE status IN ('paid', 'open') AND paid_at >= '2026-09-01' "
|
||||
"AND paid_at < '2026-09-08'",
|
||||
"SELECT COUNT(*) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '2026-09-01' AND paid_at < '2026-09-08'",
|
||||
], ids=["other-column", "or", "missing-condition", "other-operator", "number-not-string", "in-list", "other-aggregate"])
|
||||
def test_really_different_conditions_are_different_shapes(other):
|
||||
from app.data.sql_semantics import sql_shape
|
||||
|
||||
base = sql_shape(REORDERED[0].format(s="2026-09-01", e="2026-09-08"), "sqlite")
|
||||
found = sql_shape(other, "sqlite")
|
||||
assert base is not None and found is not None and found.key != base.key
|
||||
|
||||
|
||||
async def test_queue_groups_reordered_conditions_and_freezing_aligns_their_parameters(client, monkeypatch):
|
||||
a, b = await conversation(), await conversation()
|
||||
runs = [await ask(monkeypatch, conv, sql.format(s=s, e=e))
|
||||
for conv, sql, (s, e) in zip((a, a, b), REORDERED, WEEKS)]
|
||||
# 条件真不同的（多筛了一个地区）不进这一组
|
||||
other = await ask(monkeypatch, b, "SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND region = 'east' "
|
||||
"AND paid_at >= '2026-09-01' AND paid_at < '2026-09-08'")
|
||||
[item] = shape_items((await client.get("/api/ledger/certify-queue")).json())
|
||||
assert item["runs"] == 3 and {t["run_id"] for t in item["turns"]} == {r.id for r in runs}
|
||||
assert other.id not in {t["run_id"] for t in item["turns"]}
|
||||
assert len(item["params"]) == 2 and all(p["distinct"] == 3 for p in item["params"]) and item["constants"] == 1
|
||||
|
||||
rep = item["representative"]["run_id"]
|
||||
res = await client.post("/api/copilot/from-run", json={"run_id": rep, "freeze": True, "peers": item["peers"]})
|
||||
assert res.status_code == 200, res.text
|
||||
made = res.json()
|
||||
assert made["params"] == ["p1", "p2"], "条件换了序的几轮照样认得出两个日期是参数"
|
||||
async with SessionLocal() as session:
|
||||
wf = await session.get(Workflow, made["workflow_id"])
|
||||
sql = {n["id"]: n for n in wf.graph["nodes"]}["fetch"]["data"]["config"]["args"]["sql"]
|
||||
assert "'paid'" in sql and "'{{ input.p1 }}'" in sql and "'{{ input.p2 }}'" in sql
|
||||
|
||||
|
||||
async def test_a_not_a_caliber_recorded_by_the_old_text_id_still_counts(client, monkeypatch):
|
||||
"""以前「不是口径」按规范文本算的 id 记：分组改按规范形之后，组里任何一条的规范文本对得上就照样不进队。"""
|
||||
await three_asks(monkeypatch)
|
||||
[item] = shape_items((await client.get("/api/ledger/certify-queue")).json())
|
||||
old = certify.item_id("shape", f"{S.CURRENT['source']}\n{item['shape']}")
|
||||
assert old != item["id"]
|
||||
res = await client.post("/api/ledger/certify-queue/dismiss", json={"item_id": old, "kind": "shape"})
|
||||
assert res.status_code == 200
|
||||
assert shape_items((await client.get("/api/ledger/certify-queue")).json()) == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 问题聚类的 id 由成员定；「不是口径」按成员记
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_cluster_ids_come_from_the_members_and_do_not_change_between_reads():
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.chat.questions import Question, QuestionCluster, cluster_questions
|
||||
|
||||
at = datetime(2026, 10, 1, tzinfo=timezone.utc)
|
||||
# 同一时刻问的三句（时间相同，库里读出来的先后不定）：两次读出来先后不同，种子就不同，类的 id 仍然一样
|
||||
texts = ["仓库乙本月出库多少件", "仓库乙本月出库多少件？", "仓库乙本月出库了多少件"]
|
||||
questions = [Question(text=t, at=at, turn_id=f"t-{i}") for i, t in enumerate(texts)]
|
||||
first = await cluster_questions(questions)
|
||||
again = await cluster_questions(list(reversed(questions)))
|
||||
assert len(first) == 1 and len(again) == 1 and first[0].members[0].text != again[0].members[0].text
|
||||
assert [c.id for c in first] == [c.id for c in again]
|
||||
assert first[0].keys == ["turn:t-0", "turn:t-1", "turn:t-2"]
|
||||
assert QuestionCluster(members=list(reversed(questions))).id == first[0].id
|
||||
# 成员不同，id 就不同
|
||||
assert QuestionCluster(members=questions[:2]).id != first[0].id
|
||||
|
||||
|
||||
async def test_not_a_caliber_on_a_cluster_survives_a_new_member_and_old_records_still_count(client, monkeypatch):
|
||||
import uuid
|
||||
|
||||
from app.auth.audit import record
|
||||
|
||||
text = f"仓库甲本周入库了多少件（{uuid.uuid4().hex[:6]}）"
|
||||
a, b = await conversation(), await conversation()
|
||||
asked = {(await ask(monkeypatch, conv, weekly(*WEEKS[0]), question=text)).id for conv in (a, a, b)}
|
||||
|
||||
async def mine() -> list[dict]:
|
||||
body = (await client.get("/api/ledger/certify-queue")).json()
|
||||
return [i for i in body["items"] if i["kind"] == "cluster" and asked & {t["run_id"] for t in i["turns"]}]
|
||||
|
||||
[item] = await mine()
|
||||
[again] = await mine()
|
||||
assert item["id"] == again["id"], "同一批问题两次读取，类的 id 一样"
|
||||
# 读不懂的老记录照实忽略：不报错、不把这一类藏起来
|
||||
async with SessionLocal() as session:
|
||||
await record(session, certify.DISMISS_ACTION, certify.SUBJECT_KIND, "cluster:0000000000000000",
|
||||
data={"kind": "cluster", "members": "看不懂"})
|
||||
await session.commit()
|
||||
assert [i["id"] for i in await mine()] == [item["id"]]
|
||||
|
||||
res = await client.post("/api/ledger/certify-queue/dismiss", json={"item_id": item["id"], "kind": "cluster"})
|
||||
assert res.status_code == 200
|
||||
assert await mine() == []
|
||||
# 再换个说法问一句：成员多了一个、种子换成了新的这句，类的 id 变了；过半的成员是点「不是口径」时记下的，照样不进队
|
||||
newer = await ask(monkeypatch, b, weekly(*WEEKS[1]), question=text + "？")
|
||||
asked.add(newer.id)
|
||||
assert await mine() == []
|
||||
|
||||
|
||||
async def test_a_not_a_caliber_recorded_by_the_old_seed_text_id_still_counts(client, monkeypatch):
|
||||
import uuid
|
||||
|
||||
from app.auth.audit import record
|
||||
|
||||
text = f"仓库丙上周退货多少件（{uuid.uuid4().hex[:6]}）"
|
||||
a, b = await conversation(), await conversation()
|
||||
for conv in (a, a, b):
|
||||
await ask(monkeypatch, conv, weekly(*WEEKS[0]), question=text)
|
||||
body = (await client.get("/api/ledger/certify-queue")).json()
|
||||
assert [i for i in body["items"] if i["kind"] == "cluster" and i["question"] == text]
|
||||
async with SessionLocal() as session:
|
||||
await record(session, certify.DISMISS_ACTION, certify.SUBJECT_KIND, certify.item_id("cluster", text),
|
||||
data={"kind": "cluster"})
|
||||
await session.commit()
|
||||
body = (await client.get("/api/ledger/certify-queue")).json()
|
||||
assert not [i for i in body["items"] if i["kind"] == "cluster" and i["question"] == text]
|
||||
|
||||
@@ -253,7 +253,7 @@ async def test_an_answer_built_with_use_caliber_is_labelled_composed(client, mon
|
||||
await ledger_match.drain()
|
||||
recorded = await store.latest(row.id)
|
||||
content = recorded.content
|
||||
assert content["algo"] == store.ALGO == "2"
|
||||
assert content["algo"] == store.ALGO == "3"
|
||||
[query] = content["queries"]
|
||||
assert query["status"] == "composed" and query["node_id"] == "c1_fetch"
|
||||
[cand] = query["candidates"]
|
||||
@@ -292,3 +292,33 @@ async def test_users_do_not_get_calibers_in_the_answer_look(monkeypatch):
|
||||
with pytest.raises(RuntimeError):
|
||||
await copilot.answer_ops(GenerateIn(instruction="x", intent="build")).__anext__()
|
||||
assert seen["kinds"] == ("table", "capability")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# card 填什么：协议写清两个 id，写错了交回的原因能一次改对（校验不放宽）
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_protocol_says_card_is_the_cards_node_id_and_id_is_the_name_you_give():
|
||||
text = copilot_context.USE_CALIBER_PROMPT
|
||||
assert '"card":"card"' in text and '"card":"口径卡节点 id"' not in text
|
||||
assert "card 填那一版工作流图里口径卡的节点 id" in text and "不是你起的名字" in text
|
||||
assert "id 是你给这次选用起的名字" in text and "workflow_id、version、card 原样照抄" in text
|
||||
|
||||
|
||||
async def test_writing_your_own_id_as_the_card_is_refused_with_what_to_write_instead():
|
||||
wf = await S.publish()
|
||||
pool = await composer()
|
||||
with pytest.raises(compose.ComposeRefused) as e:
|
||||
pool.compose(use(wf, card="c1"), taken={"start"})
|
||||
message = e.value.message
|
||||
assert "没有口径卡「c1」" in message and "card 要填那一版工作流图里口径卡的节点 id" in message
|
||||
assert "可填的有 「card」（口径卡「card」）" in message, message
|
||||
assert "「c1」是你给这次选用起的 id，不是口径卡的节点 id" in message
|
||||
# 别的写错（不是自己起的 id）不多那一句
|
||||
with pytest.raises(compose.ComposeRefused) as other:
|
||||
pool.compose(use(wf, card="other"), taken={"start"})
|
||||
assert "是你给这次选用起的 id" not in other.value.message
|
||||
# 校验不放宽：照着交回的原因改成节点 id 就过
|
||||
made = pool.compose(use(wf, card="card"), taken={"start"})
|
||||
assert made.op_id == "c1"
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
"""认口径时按引用的列投影（app/ledger/match.py 的 _projected，algo 3）。
|
||||
|
||||
真实助手的 SQL 总会多选几列（订单数、日期常量、MIN / MAX……），整条比永远不等价，「上周的销售额是多少」从不自动出具。
|
||||
整条比不等价时，答案一侧按报告引用的列、口径一侧按口径卡上指标读的列投影，再交给现有的绑定和判等价。这里守住：
|
||||
- 多选一列、多几个聚合、日期常量列的，按引用的那一列证明等价；口径一侧多选了列的同样认得出；留下两列先后不同的另试调换；
|
||||
- 不错给信任：多一个分组列、DISTINCT、按别的列排序再取前几行、HAVING 用到多出来的聚合的，一律仍然证明不了，结果和
|
||||
不投影时一字不差；
|
||||
- 没投影的（报告没引用、两边都不用投影、投影不了）逐对和以前一字不差；
|
||||
- 没证明的，差在哪按投影之后说：先说「这次多查了 N 列，不影响这个数」，再照常说条件上的不同；
|
||||
- 跑一次真的探索运行：报告引用销售额那一格，整题等价；E1 照实数出几份是投影之后才证明的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.engine.runner import run_manager
|
||||
from app.ledger import match as ledger_match
|
||||
from app.ledger import stats as ledger_stats
|
||||
from app.ledger import store
|
||||
from tests import ledger_samples as S
|
||||
|
||||
WINDOW = "status = 'paid' AND paid_at >= '2026-09-01' AND paid_at < '2026-09-08'"
|
||||
|
||||
|
||||
def caliber(sql: str = S.TEMPLATE, metrics: list[dict] | None = None) -> ledger_match.Caliber:
|
||||
graph = S.caliber_graph(sql)
|
||||
if metrics is not None:
|
||||
graph["nodes"][2]["data"]["config"]["metrics"] = metrics
|
||||
found = ledger_match.caliber_of("wf", SimpleNamespace(graph=graph, version=1, level="governed", graph_hash="h"))
|
||||
assert found is not None
|
||||
return found
|
||||
|
||||
|
||||
def query(sql: str, columns: list[str], artifact: str = "q1") -> ledger_match.Query:
|
||||
return ledger_match.Query(artifact, "q", S.CURRENT["source"], sql, columns, frozenset({"orders"}), "sqlite")
|
||||
|
||||
|
||||
def compare(sql: str, columns: list[str], cited: set[str] | None, cal: ledger_match.Caliber | None = None) -> dict:
|
||||
[row] = ledger_match.compare_queries([query(sql, columns)], [cal or caliber()],
|
||||
referenced={"q1": cited} if cited is not None else None)
|
||||
return row
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 认得出
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, columns", [
|
||||
(f"SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数 FROM orders WHERE {WINDOW}", ["销售额", "单数"]),
|
||||
(f"SELECT COUNT(*) AS 单数, MIN(paid_at) AS 起, MAX(paid_at) AS 止, SUM(amount) AS 销售额, "
|
||||
f"AVG(amount) AS 客单价 FROM orders WHERE {WINDOW}", ["单数", "起", "止", "销售额", "客单价"]),
|
||||
(f"SELECT '2026-09-01' AS 周一, '2026-09-07' AS 周日, SUM(amount) AS 销售额 FROM orders WHERE {WINDOW}",
|
||||
["周一", "周日", "销售额"]),
|
||||
], ids=["extra-count", "extra-aggregates", "date-constants"])
|
||||
def test_extra_columns_are_projected_away_and_the_cited_one_is_proven(sql, columns):
|
||||
row = compare(sql, columns, {"销售额"})
|
||||
assert row["status"] == "proven", row
|
||||
[cand] = row["candidates"]
|
||||
assert cand["status"] == "proven" and cand["binding"] == {"start": "2026-09-01", "end": "2026-09-08"}
|
||||
assert cand["columns"] == [{"index": 0, "ours": "销售额", "caliber": "gmv"}]
|
||||
assert cand["projection"] == {"ours": ["销售额"], "dropped": len(columns) - 1, "caliber": ["gmv"],
|
||||
"caliber_dropped": 0}
|
||||
# 不投影（报告没说引用了哪一列）时整条比，照旧证明不了
|
||||
assert compare(sql, columns, None)["status"] == "related"
|
||||
|
||||
|
||||
def test_the_caliber_side_is_projected_to_the_columns_its_metrics_read():
|
||||
two = S.TEMPLATE.replace("SUM(amount) AS gmv", "SUM(amount) AS gmv, COUNT(*) AS n")
|
||||
cal = caliber(two, [{"id": "gmv", "name": "销售额", "expression": "cell(nodes.fetch, 0, 'gmv')"},
|
||||
{"id": "n", "name": "单数", "expression": "cell(nodes.fetch, 0, 'n')"}])
|
||||
row = compare(f"SELECT SUM(amount) AS 销售额 FROM orders WHERE {WINDOW}", ["销售额"], {"销售额"}, cal)
|
||||
assert row["status"] == "proven"
|
||||
[cand] = row["candidates"]
|
||||
assert cand["columns"] == [{"index": 0, "ours": "销售额", "caliber": "gmv"}]
|
||||
assert cand["projection"] == {"ours": ["销售额"], "dropped": 0, "caliber": ["gmv"], "caliber_dropped": 1}
|
||||
# 两列都引用、先后和口径的不同：调换之后按位置对上
|
||||
row = compare(f"SELECT COUNT(*) AS 单数, SUM(amount) AS 销售额, MAX(amount) AS 最大 FROM orders WHERE {WINDOW}",
|
||||
["单数", "销售额", "最大"], {"单数", "销售额"}, cal)
|
||||
assert row["status"] == "proven"
|
||||
[cand] = row["candidates"]
|
||||
assert [(c["ours"], c["caliber"]) for c in cand["columns"]] == [("销售额", "gmv"), ("单数", "n")]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 不错给信任
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
GROUPED = S.TEMPLATE.replace("SUM(amount) AS gmv", "DISTINCT amount AS gmv")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, columns, cited, cal_sql", [
|
||||
# 多一个分组列：答案是每个地区一行，口径只有一行
|
||||
(f"SELECT region, SUM(amount) AS 销售额 FROM orders WHERE {WINDOW} GROUP BY region", ["region", "销售额"],
|
||||
{"销售额"}, S.TEMPLATE),
|
||||
# DISTINCT 按整行去重：去掉 region 之后行数就变了（口径按金额去重，投影了就会错认）
|
||||
(f"SELECT DISTINCT region, amount AS 金额 FROM orders WHERE {WINDOW}", ["region", "金额"], {"金额"}, GROUPED),
|
||||
# 按别的列排序再取前几行
|
||||
(f"SELECT SUM(amount) AS 销售额, region FROM orders WHERE {WINDOW} GROUP BY region ORDER BY region LIMIT 1",
|
||||
["销售额", "region"], {"销售额"}, S.TEMPLATE),
|
||||
(f"SELECT amount AS 金额, paid_at FROM orders WHERE {WINDOW} ORDER BY paid_at DESC LIMIT 1", ["金额", "paid_at"],
|
||||
{"金额"}, S.TEMPLATE.replace("SUM(amount) AS gmv", "amount AS gmv") + " ORDER BY amount DESC LIMIT 1"),
|
||||
# HAVING 用到多出来的聚合（按别名、按同样的写法）
|
||||
(f"SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数 FROM orders WHERE {WINDOW} HAVING 单数 > 0", ["销售额", "单数"],
|
||||
{"销售额"}, S.TEMPLATE),
|
||||
(f"SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数 FROM orders WHERE {WINDOW} HAVING COUNT(*) > 0",
|
||||
["销售额", "单数"], {"销售额"}, S.TEMPLATE),
|
||||
], ids=["extra-group-by", "distinct", "group-order-limit", "order-other-column-limit", "having-alias",
|
||||
"having-same-aggregate"])
|
||||
def test_projection_never_proves_what_the_whole_sql_does_not_say(sql, columns, cited, cal_sql):
|
||||
expression = "cell(nodes.fetch, 0, 'gmv')"
|
||||
cal = caliber(cal_sql, [{"id": "gmv", "name": "销售额", "expression": expression}])
|
||||
row = compare(sql, columns, cited, cal)
|
||||
assert row["status"] != "proven", row
|
||||
assert all("projection" not in c for c in row["candidates"]), "这几种都不该投影"
|
||||
# 不投影的：和没给引用列时一字不差
|
||||
assert row == compare(sql, columns, None, cal)
|
||||
|
||||
|
||||
def test_without_a_projection_every_pair_is_exactly_the_old_result():
|
||||
"""报告引用了全部的列、口径也不用投影的，整条比证明了的，投影不了的：逐对和不给引用列时一字不差。"""
|
||||
cases = [
|
||||
(S.EQUIVALENT, ["销售额"], {"销售额"}), # 整条比就证明了
|
||||
(S.WITH_REFUNDS, ["gmv"], {"gmv"}), # 两边都只有这一列：没有可投影的
|
||||
(f"SELECT SUM(amount) AS gmv, COUNT(*) AS n FROM orders WHERE {WINDOW}", ["gmv", "n"], {"gmv", "n"}),
|
||||
(f"SELECT DISTINCT SUM(amount) AS gmv, COUNT(*) AS n FROM orders WHERE {WINDOW}", ["gmv", "n"], {"gmv"}),
|
||||
(f"SELECT SUM(amount) AS gmv, COUNT(*) AS n FROM orders WHERE {WINDOW}", ["gmv"], {"gmv"}), # 列数对不上
|
||||
]
|
||||
for sql, columns, cited in cases:
|
||||
assert compare(sql, columns, cited) == compare(sql, columns, None), sql
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 说得清：没证明的按投影之后说
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_an_unproven_projection_says_the_extra_columns_first_then_the_real_difference():
|
||||
sql = ("SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数, MIN(paid_at) AS 起, MAX(paid_at) AS 止 FROM orders "
|
||||
"WHERE paid_at >= '2026-09-01' AND paid_at < '2026-09-08'")
|
||||
row = compare(sql, ["销售额", "单数", "起", "止"], {"销售额"})
|
||||
assert row["status"] == "related"
|
||||
[cand] = row["candidates"]
|
||||
items = cand["diff"]["items"]
|
||||
assert items[0] == {"aspect": "projection", "a": None, "b": None, "text": "这次多查了 3 列，不影响这个数"}
|
||||
assert items[1]["aspect"] == "filter" and "status = 'paid'" in items[1]["text"]
|
||||
assert cand["diff"]["fallback"] is False
|
||||
assert cand["projection"]["dropped"] == 3
|
||||
# 不投影时整段退回兜底（真实验收里 E2 的兜底比例 45% 就是这么来的）
|
||||
[old] = compare(sql, ["销售额", "单数", "起", "止"], None)["candidates"]
|
||||
assert old["diff"]["fallback"] is True
|
||||
|
||||
|
||||
def test_e1_does_not_count_the_extra_columns_note_as_a_difference_and_counts_projected_proofs():
|
||||
proven = compare(f"SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数 FROM orders WHERE {WINDOW}",
|
||||
["销售额", "单数"], {"销售额"})
|
||||
related = compare("SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数 FROM orders WHERE paid_at >= '2026-09-01' "
|
||||
"AND paid_at < '2026-09-08'", ["销售额", "单数"], {"销售额"})
|
||||
e1 = ledger_stats.e1_of([{"queries": [proven, related], "reports": {}, "whole": {}}])
|
||||
assert e1["status"]["proven"] == 1 and e1["proven_projected"] == 1
|
||||
assert e1["diff_aspects"]["filter"] == 1 and e1["diff_aspects"]["other"] == 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 真的跑一遍
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def engine_up(tmp_path):
|
||||
await run_manager.setup()
|
||||
await S.fresh_source(tmp_path / "shop.db")
|
||||
yield
|
||||
await ledger_match.drain()
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
async def test_last_weeks_sales_with_extra_columns_is_whole_equivalent(engine_up, monkeypatch):
|
||||
wf = await S.publish()
|
||||
sql = ("SELECT '2026-09-01' AS 周一, SUM(amount) AS 销售额, COUNT(*) AS 单数, MAX(paid_at) AS 最后一单 "
|
||||
f"FROM orders WHERE {WINDOW}")
|
||||
row = await S.explore(monkeypatch, sql, text="上周销售额 [[v:Q1.r0.销售额]]，共 [[v:Q1.r0.单数]] 单。")
|
||||
recorded = await store.latest(row.id)
|
||||
content = recorded.content
|
||||
assert content["algo"] == "3"
|
||||
[q] = content["queries"]
|
||||
assert q["status"] == "related", "销售额、单数两列都被引用，口径只有销售额：投影之后列数对不上，照旧不等价"
|
||||
|
||||
row = await S.explore(monkeypatch, sql, text="上周销售额 [[v:Q1.r0.销售额]]。")
|
||||
content = (await store.latest(row.id)).content
|
||||
[q] = content["queries"]
|
||||
assert q["status"] == "proven"
|
||||
# 评测逐题比 SQL 时拿同一份「引用了哪几列」（referenced_of），和账本一样投影
|
||||
from app.api.evidence import _sealed
|
||||
|
||||
assert ledger_match.referenced_of(await _sealed(row.id)) == {q["artifact"]: {"销售额"}}
|
||||
[cand] = q["candidates"]
|
||||
assert cand["projection"]["dropped"] == 3 and cand["columns"] == [{"index": 0, "ours": "销售额", "caliber": "gmv"}]
|
||||
[labels] = [r["labels"] for r in content["reports"].values()]
|
||||
[label] = labels.values()
|
||||
assert label["kind"] == "caliber" and label["column"] == "gmv" and label["caliber"] == f"{wf}@1"
|
||||
assert content["whole"]["equivalent"] is True and content["whole"]["binding"] == {"start": "2026-09-01",
|
||||
"end": "2026-09-08"}
|
||||
|
||||
|
||||
async def test_reconcile_pairs_the_columns_left_after_the_projection(engine_up, monkeypatch):
|
||||
"""含退款、又多选了几列的答案：对账时按投影之后两边留下的列配（这次的销售额 → 口径卡的 gmv），不用人选。"""
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.main import app
|
||||
|
||||
wf = await S.publish()
|
||||
sql = ("SELECT COUNT(*) AS 单数, SUM(amount) AS 销售额, MAX(paid_at) AS 最后一单 FROM orders "
|
||||
"WHERE paid_at >= '2026-09-01' AND paid_at < '2026-09-08'")
|
||||
row = await S.explore(monkeypatch, sql, text="上周销售额 [[v:Q1.r0.销售额]]。")
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||
body = (await client.get(f"/api/runs/{row.id}/ledger")).json()
|
||||
[q] = body["queries"]
|
||||
[cand] = q["candidates"]
|
||||
assert cand["projection"]["ours"] == ["销售额"] and cand["projection"]["caliber"] == ["gmv"]
|
||||
assert cand["diff"]["items"][0]["text"] == "这次多查了 2 列，不影响这个数"
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile", json={
|
||||
"query": q["artifact"], "workflow_id": wf, "version": 1, "confirm": True,
|
||||
"inputs": {"start": "2026-09-01", "end": "2026-09-08"}})
|
||||
assert res.status_code == 201, res.text
|
||||
done = await S.wait(res.json()["run_id"])
|
||||
assert done.status == "succeeded", done.error
|
||||
assert done.origin["pairs"] == [{"ours": "销售额", "metric": "gmv"}]
|
||||
@@ -0,0 +1,196 @@
|
||||
"""转抄另一张口径卡的指标时，出处链跨卡往下追（engine/lineage.metric_copy、engine/nodes/metrics._upstream、
|
||||
api/evidence._upstream_steps）。
|
||||
|
||||
对账卡的「口径」、召回卡的「旧版 / 新版」都写成 nodes.<口径卡>.metrics[i].value。以前出处停在「来自口径卡指标」，对账运行、
|
||||
召回运行一律降档出具（「输入没追到原始数据」）；口径卡自己的指标明明追得到取数查询。现在：
|
||||
- 运行时接上那个指标自己的出处（upstream）：那个指标的输入都追到了原始数据，这一项才算追到，判档按追到底的结果；
|
||||
那个指标不完整、所依据的查询没通过 SQL 检查的，转抄出来的同样不完整、存疑；转抄的转抄一层层往下追，到了层数上限
|
||||
就算没追到；
|
||||
- 证据接口：转抄的输入后面接一步 upstream（那张卡的那个指标、它的输入），一直到口径的取数查询（查询步骤照常给）；
|
||||
- 真跑对账、召回：完整出具，三个数都点得到口径的取数查询。
|
||||
数据是虚构店铺库。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.engine.lineage import MAX_COPY_DEPTH, traced, untraced
|
||||
from app.engine.runner import run_manager
|
||||
from app.ledger import match as ledger_match
|
||||
from app.ledger import recall
|
||||
from tests import ledger_samples as S
|
||||
|
||||
WINDOW = "status = 'paid' AND paid_at >= '2026-09-01' AND paid_at < '2026-09-08'"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up(tmp_path):
|
||||
await run_manager.setup()
|
||||
await S.fresh_source(tmp_path / "shop.db")
|
||||
yield
|
||||
await ledger_match.drain()
|
||||
await recall.drain()
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
from app.main import app
|
||||
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
def metric(mid: str, expression: str, name: str = "") -> dict:
|
||||
return {"id": mid, "name": name or mid, "unit": "元", "decimals": 2, "expression": expression}
|
||||
|
||||
|
||||
async def run_chain(*middle: dict, contract_from: str) -> object:
|
||||
graph = S.chain(S.node("start", "input"), *middle,
|
||||
S.node("out", "output", fields=[{"name": "数", "value": f"{{{{ nodes.{contract_from}.metrics[0].value }}}}"}],
|
||||
contract={"metrics_from": contract_from}))
|
||||
run = await run_manager.start(graph=graph, input_payload={})
|
||||
row = await S.wait(run.id)
|
||||
assert row.status == "succeeded", row.error
|
||||
return row
|
||||
|
||||
|
||||
async def card(run_id: str, node_id: str) -> dict:
|
||||
from app.api.evidence import _sealed, _sealed_cards
|
||||
|
||||
sealed = await _sealed(run_id)
|
||||
[(_, found)] = [(a, c) for n, a, c in _sealed_cards(sealed) if n == node_id]
|
||||
return {m["id"]: m for m in found["metrics"]}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 运行时
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_copied_metric_traces_through_the_card_and_issues_formally():
|
||||
fetch = S.node("fetch", "tool", tool=S.tool(), args={"sql": f"SELECT SUM(amount) AS gmv FROM orders WHERE {WINDOW}"})
|
||||
a = S.node("a", "metrics", caliber="口径", caliber_version="v1", metrics=[metric("gmv", "cell(nodes.fetch, 0, 'gmv')")])
|
||||
b = S.node("b", "metrics", caliber="转抄", caliber_version="v1", metrics=[
|
||||
metric("cal", "nodes.a.metrics[0].value"), metric("half", "(nodes.a.metrics[0].value) / 2")])
|
||||
c = S.node("c", "metrics", caliber="再转抄", caliber_version="v1", metrics=[metric("again", "nodes.b.metrics[0].value")])
|
||||
row = await run_chain(fetch, a, b, c, contract_from="c")
|
||||
first, second = await card(row.id, "b"), await card(row.id, "c")
|
||||
[item] = first["cal"]["inputs"]
|
||||
assert item["via"] == "metric" and item["node_id"] == "a"
|
||||
assert item["upstream"]["traced"] is True and item["upstream"]["depth"] == 1 and item["upstream"]["metric"] == "gmv"
|
||||
assert traced(item) and untraced(first["half"]["inputs"]) == []
|
||||
[deep] = second["again"]["inputs"]
|
||||
assert deep["upstream"]["traced"] is True and deep["upstream"]["depth"] == 2
|
||||
issuance = row.output["_issuance"]
|
||||
assert issuance["tier"] == "formal", issuance["gaps"]
|
||||
|
||||
|
||||
async def test_copying_a_metric_that_is_not_traced_still_stops_there():
|
||||
calc = S.node("calc", "transform", mode="json", template='{"v": 5}', assign_to="calc")
|
||||
a = S.node("a", "metrics", caliber="口径", caliber_version="v1", metrics=[metric("five", "vars.calc.v")])
|
||||
b = S.node("b", "metrics", caliber="转抄", caliber_version="v1", metrics=[metric("cal", "nodes.a.metrics[0].value")])
|
||||
row = await run_chain(calc, a, b, contract_from="b")
|
||||
[item] = (await card(row.id, "b"))["cal"]["inputs"]
|
||||
assert item["upstream"]["traced"] is False and not traced(item)
|
||||
issuance = row.output["_issuance"]
|
||||
assert issuance["tier"] == "degraded"
|
||||
assert any("指标「cal」的输入没追到原始数据" in g for g in issuance["gaps"]), issuance["gaps"]
|
||||
|
||||
|
||||
async def test_an_incomplete_upstream_metric_makes_the_copy_incomplete_too():
|
||||
fetch = S.node("fetch", "tool", tool=S.tool(), args={"sql": "SELECT id, amount FROM orders ORDER BY id", "limit": 2})
|
||||
parse = S.node("parse", "transform", mode="json", template="{{ nodes.fetch }}", assign_to="rows")
|
||||
a = S.node("a", "metrics", caliber="口径", caliber_version="v1", metrics=[metric("n", "len(vars.rows.rows)")])
|
||||
b = S.node("b", "metrics", caliber="转抄", caliber_version="v1", metrics=[metric("cal", "nodes.a.metrics[0].value")])
|
||||
row = await run_chain(fetch, parse, a, b, contract_from="b")
|
||||
upstream, copied = (await card(row.id, "a"))["n"], (await card(row.id, "b"))["cal"]
|
||||
assert upstream["incomplete"] is True
|
||||
assert copied["incomplete"] is True and copied["truncated_sources"] == upstream["truncated_sources"]
|
||||
assert copied["incomplete_reason"] == upstream["incomplete_reason"]
|
||||
issuance = row.output["_issuance"]
|
||||
assert issuance["tier"] == "degraded"
|
||||
assert any(g.startswith("指标「cal」基于被截断的查询结果计算") for g in issuance["gaps"]), issuance["gaps"]
|
||||
|
||||
|
||||
def test_copies_of_copies_stop_at_the_depth_limit():
|
||||
from app.engine.nodes.metrics import _upstream
|
||||
|
||||
def card_with(depth: int) -> dict:
|
||||
inputs = [{"path": "nodes.x.metrics[0].value", "value": 1, "via": "metric", "node_id": "x",
|
||||
"upstream": {"metric": "m", "artifact": "a" * 64, "depth": depth, "traced": True}}]
|
||||
return {"kind": "metric_set", "artifact": "b" * 64, "metrics": [{"id": "m", "value": 1, "inputs": inputs}]}
|
||||
|
||||
ok = _upstream("nodes.y.metrics[0].value", {"nodes": {"y": card_with(MAX_COPY_DEPTH - 1)}})
|
||||
assert ok is not None and ok["traced"] is True and ok["depth"] == MAX_COPY_DEPTH
|
||||
too_deep = _upstream("nodes.y.metrics[0].value", {"nodes": {"y": card_with(MAX_COPY_DEPTH)}})
|
||||
assert too_deep is not None and too_deep["traced"] is False and too_deep["depth"] == MAX_COPY_DEPTH + 1
|
||||
# 整张卡、清单文字、指标的名称这类不是转抄一个指标的值：不往下追
|
||||
assert _upstream("nodes.y.metrics", {"nodes": {"y": card_with(0)}}) is None
|
||||
assert _upstream("nodes.y.metrics[0].name", {"nodes": {"y": card_with(0)}}) is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 真跑对账、召回：完整出具，证据面板一路点到口径的取数查询
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _steps(chain: list[dict], kind: str) -> list[dict]:
|
||||
return [s for s in chain if s["step"] == kind]
|
||||
|
||||
|
||||
async def test_a_reconcile_run_traces_the_caliber_side_down_to_its_fetch(client, monkeypatch):
|
||||
wf = await S.publish()
|
||||
row = await S.explore(monkeypatch, S.WITH_REFUNDS)
|
||||
body = (await client.get(f"/api/runs/{row.id}/ledger")).json()
|
||||
[action] = [a for a in body["actions"] if a["kind"] == "reconcile"]
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile", json={
|
||||
"query": action["query"], "workflow_id": wf, "version": 1, "confirm": True,
|
||||
"inputs": {"start": "2026-09-01", "end": "2026-09-08"}})
|
||||
assert res.status_code == 201, res.text
|
||||
done = await S.wait(res.json()["run_id"])
|
||||
assert done.status == "succeeded", done.error
|
||||
issuance = done.output["_issuance"]
|
||||
assert issuance["tier"] == "formal", issuance
|
||||
metrics = await card(done.id, "reconcile")
|
||||
[cal] = metrics["cal_0"]["inputs"]
|
||||
assert cal["node_id"] == "card" and cal["upstream"]["traced"] is True
|
||||
|
||||
for mid in ("cal_0", "diff_0"):
|
||||
res = await client.get(f"/api/runs/{done.id}/evidence/metrics/reconcile/{mid}")
|
||||
assert res.status_code == 200, res.text
|
||||
chain = res.json()["chain"]
|
||||
assert chain[0]["untraced"] == [], "转抄的口径那一项跨卡追到了原始数据"
|
||||
[up] = _steps(chain, "upstream")
|
||||
assert up["of"] == "nodes.card.metrics[0].value" and up["node_id"] == "card" and up["metric"] == "gmv"
|
||||
assert up["traced"] is True and up["sealed"] is True and up["hash_ok"] is True and up["value"] == 170.5
|
||||
[read] = up["inputs"]
|
||||
assert read["path"] == "cell(nodes.fetch, 0, 'gmv')" and read["via"] == "tool_cell"
|
||||
# 口径的取数查询的那一步照常给（点得到）
|
||||
fetches = [q for q in _steps(chain, "query") if q.get("artifact") == read["artifact"]]
|
||||
assert len(fetches) == 1 and fetches[0]["node_id"] == "fetch"
|
||||
# 「这次」那一项不是转抄的，没有 upstream 这一步
|
||||
ours = (await client.get(f"/api/runs/{done.id}/evidence/metrics/reconcile/ours_0")).json()["chain"]
|
||||
assert _steps(ours, "upstream") == []
|
||||
|
||||
|
||||
async def test_a_recall_run_traces_both_versions_down_to_their_fetches(client, monkeypatch):
|
||||
from tests.test_ledger_recall import WITH_REFUNDED, answer, ledger_settings, publish_v2, run_recall
|
||||
|
||||
await ledger_settings(auto_issue=False, issue_daily_limit=100_000)
|
||||
wf = await S.publish()
|
||||
await answer(monkeypatch)
|
||||
await publish_v2(wf, WITH_REFUNDED)
|
||||
[done] = await run_recall(wf)
|
||||
assert done.status == "succeeded", done.error
|
||||
assert done.output["_issuance"]["tier"] == "formal", done.output["_issuance"]
|
||||
res = await client.get(f"/api/runs/{done.id}/evidence/metrics/recall/diff_0")
|
||||
assert res.status_code == 200, res.text
|
||||
chain = res.json()["chain"]
|
||||
ups = {u["node_id"]: u for u in _steps(chain, "upstream")}
|
||||
assert set(ups) == {"old_card", "new_card"}
|
||||
assert all(u["traced"] for u in ups.values())
|
||||
reads = {u["node_id"]: u["inputs"][0]["artifact"] for u in ups.values()}
|
||||
queried = {q["node_id"] for q in _steps(chain, "query") if q.get("artifact") in reads.values()}
|
||||
assert queried == {"old_fetch", "new_fetch"}
|
||||
@@ -0,0 +1,82 @@
|
||||
"""日期条件写在 LEFT JOIN 的 ON 里时的差异说明（data/sql_semantics.py 的 _Explainer._left_on_times）。
|
||||
|
||||
以前两边来源一样、只差在一个 LEFT JOIN 的 ON 里的日期条件时，整段退回「两条 SQL 的写法不同，未能归纳成具体差异」。现在
|
||||
说得准的照实说（aspect 是 join：说的是关联条件，不会被当成业务日期写进数据目录），说不准的照实兜底。只改说明：判等价的
|
||||
结论一个不变；说出来的不同在虚构库上确实让结果不一样。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.data.sql_semantics import explain_sql_diff
|
||||
from tests.fixtures import sql_pairs
|
||||
|
||||
SCHEMA = sql_pairs.schema_cache()
|
||||
BASE = "SELECT u.city, SUM(o.amount) AS s FROM users u LEFT JOIN orders o ON o.user_id = u.id{on} GROUP BY u.city"
|
||||
|
||||
SAID = [
|
||||
# 一边多一个日期条件
|
||||
(BASE.format(on=" AND o.paid_at >= '2026-09-10'"), BASE.format(on=""),
|
||||
"这次用 LEFT JOIN 关联 orders 表时只关联 orders.paid_at >= '2026-09-10' 的行，口径没有这个条件"),
|
||||
(BASE.format(on=""), BASE.format(on=" AND o.created_at < '2026-09-20'"),
|
||||
"口径用 LEFT JOIN 关联 orders 表时只关联 orders.created_at < '2026-09-20' 的行，这次没有这个条件"),
|
||||
# 按不同的时间列筛
|
||||
(BASE.format(on=" AND o.paid_at >= '2026-09-10'"), BASE.format(on=" AND o.created_at >= '2026-09-10'"),
|
||||
"这次用 LEFT JOIN 关联 orders 表时按 paid_at 筛时间，口径按 created_at"),
|
||||
# 同一列、范围不同
|
||||
(BASE.format(on=" AND o.paid_at >= '2026-09-10' AND o.paid_at < '2026-10-01'"),
|
||||
BASE.format(on=" AND o.paid_at >= '2026-09-01' AND o.paid_at < '2026-10-01'"),
|
||||
"这次用 LEFT JOIN 关联 orders 表时对 paid_at 的条件是 orders.paid_at >= '2026-09-10' AND orders.paid_at < "
|
||||
"'2026-10-01'，口径是 orders.paid_at >= '2026-09-01' AND orders.paid_at < '2026-10-01'"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def databases():
|
||||
conns = sql_pairs.build_databases()
|
||||
yield conns
|
||||
for c in conns:
|
||||
c.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("a, b, text", SAID, ids=["extra-on-ours", "extra-on-caliber", "other-column", "range"])
|
||||
@pytest.mark.parametrize("schema", [None, SCHEMA], ids=["bare", "schema"])
|
||||
def test_date_conditions_in_a_left_join_on_are_said(a, b, text, schema, databases):
|
||||
diff = explain_sql_diff(a, b, "sqlite", schema=schema, labels=("这次", "口径"))
|
||||
assert diff.equivalent is False and diff.fallback is False
|
||||
assert [(i.aspect, i.text) for i in diff.items] == [("join", text)]
|
||||
# 说出来的不同是真的：在虚构库的某组数据上两条结果确实不一样
|
||||
assert any(sql_pairs.run(c, a) != sql_pairs.run(c, b) for c in databases)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("a, b", [
|
||||
# 套着日期函数，按取值认不准两边是不是一样：照实兜底
|
||||
(BASE.format(on=" AND DATE(o.paid_at) >= '2026-09-10' AND DATE(o.paid_at) <= '2026-09-30'"),
|
||||
BASE.format(on=" AND o.paid_at >= '2026-09-10' AND o.paid_at < '2026-10-01'")),
|
||||
# 可空一侧在 WHERE 里用到了（反连接）：说的就不是时间窗口了
|
||||
("SELECT u.city, COUNT(*) AS n FROM users u LEFT JOIN orders o ON o.user_id = u.id AND o.paid_at >= '2026-09-10' "
|
||||
"WHERE o.id IS NULL GROUP BY u.city",
|
||||
"SELECT u.city, COUNT(*) AS n FROM users u LEFT JOIN orders o ON o.user_id = u.id WHERE o.id IS NULL "
|
||||
"GROUP BY u.city"),
|
||||
# 结果里没用到可空一侧的列：ON 里的不同只影响行会不会重复，说不准
|
||||
("SELECT u.city, COUNT(*) AS n FROM users u LEFT JOIN orders o ON o.user_id = u.id AND o.paid_at >= '2026-09-10' "
|
||||
"GROUP BY u.city",
|
||||
"SELECT u.city, COUNT(*) AS n FROM users u LEFT JOIN orders o ON o.user_id = u.id GROUP BY u.city"),
|
||||
# ON 里多出来的不是时间条件
|
||||
(BASE.format(on=" AND o.status = 'paid'"), BASE.format(on="")),
|
||||
], ids=["date-function", "anti-join", "not-in-output", "not-time"])
|
||||
def test_what_cannot_be_said_precisely_still_gets_the_fallback(a, b):
|
||||
diff = explain_sql_diff(a, b, "sqlite", labels=("这次", "口径"))
|
||||
assert diff.equivalent is False and diff.fallback is True
|
||||
assert not [i for i in diff.items if i.aspect == "join"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("a, b, same", [
|
||||
(BASE.format(on=" AND o.paid_at >= '2026-09-10'"), BASE.format(on=" AND o.paid_at >= '2026-09-10'"), True),
|
||||
("SELECT u.city, SUM(o.amount) AS s FROM users u LEFT JOIN orders o ON o.paid_at >= '2026-09-10' AND "
|
||||
"o.user_id = u.id GROUP BY u.city", BASE.format(on=" AND o.paid_at >= '2026-09-10'"), True),
|
||||
*[(a, b, False) for a, b, _ in SAID],
|
||||
])
|
||||
def test_only_the_explanation_changed_not_the_verdict(a, b, same):
|
||||
for schema in (None, SCHEMA):
|
||||
assert explain_sql_diff(a, b, "sqlite", schema=schema).equivalent is same
|
||||
@@ -0,0 +1,195 @@
|
||||
"""按引用的列投影（data/sql_semantics.py 的 project_select / project_template / output_names，next-round 口径账本：整题等价
|
||||
按答案引用的那几列比）。
|
||||
|
||||
投影本身不判等价，只把「没被引用、去掉也不改变被引用那几列取值」的选择项去掉，交给现有的绑定和判等价。这里守两件事：
|
||||
- 该去的去得掉：多选一列、多几个聚合、日期常量列、带 WITH 的、只出一行的聚合带 LIMIT；口径模板里的 {{ input.x }} 原样留着；
|
||||
- 去了会变的一律不去（宁可不投影）：DISTINCT / DISTINCT ON、出现在 GROUP BY 里、被 ORDER BY / HAVING / QUALIFY / 窗口
|
||||
用到（按别名、按同样的写法、按位置）、和 LIMIT / OFFSET 一起定取哪几行、集合运算、*、可能改变行数的函数、去掉仅有的聚合、
|
||||
留下按任意一行取值的列。
|
||||
|
||||
另有执行守门：语料（tests/fixtures/sql_pairs.py）里每一条 SQL，按每一列（和「只去掉一列」）投影，投影得了的在虚构库的每组
|
||||
数据上，投影后的结果都得等于原结果的那几列。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.data.sql_semantics import output_names, project_select, project_template
|
||||
from tests.fixtures import sql_pairs
|
||||
|
||||
WEEKLY = ("SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' "
|
||||
"AND paid_at >= '{{ input.start }}' AND paid_at < '{{ input.end }}'")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 该去的去得掉
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, keep, want", [
|
||||
("SELECT SUM(amount) AS sales, COUNT(*) AS n FROM orders WHERE status = 'paid'", [0],
|
||||
"SELECT SUM(amount) AS sales FROM orders WHERE status = 'paid'"),
|
||||
("SELECT COUNT(*) AS n, MIN(paid_at) AS first_day, MAX(paid_at) AS last_day, SUM(amount) AS sales "
|
||||
"FROM orders WHERE paid_at >= '2026-10-05'", [3],
|
||||
"SELECT SUM(amount) AS sales FROM orders WHERE paid_at >= '2026-10-05'"),
|
||||
("SELECT '2026-10-05' AS week_start, '2026-10-11' AS week_end, COALESCE(SUM(amount), 0) AS sales FROM orders", [2],
|
||||
"SELECT COALESCE(SUM(amount), 0) AS sales FROM orders"),
|
||||
("WITH paid AS (SELECT amount FROM orders WHERE status = 'paid') SELECT SUM(amount) AS s, COUNT(*) AS n FROM paid",
|
||||
[0], "WITH paid AS (SELECT amount FROM orders WHERE status = 'paid') SELECT SUM(amount) AS s FROM paid"),
|
||||
# 不分组的聚合只出一行：LIMIT 取哪一行和选出来的列无关
|
||||
("SELECT SUM(amount) AS s, COUNT(*) AS n FROM orders LIMIT 100", [0], "SELECT SUM(amount) AS s FROM orders LIMIT 100"),
|
||||
# 分组的一段去掉别的聚合（分组键留着）
|
||||
("SELECT region, SUM(amount) AS s, COUNT(*) AS n FROM orders GROUP BY region", [0, 1],
|
||||
"SELECT region, SUM(amount) AS s FROM orders GROUP BY region"),
|
||||
# 调换先后
|
||||
("SELECT COUNT(*) AS n, SUM(amount) AS s FROM orders", [1, 0], "SELECT SUM(amount) AS s, COUNT(*) AS n FROM orders"),
|
||||
], ids=["extra-count", "extra-aggregates", "date-constants", "cte", "single-row-limit", "grouped", "reorder"])
|
||||
def test_items_that_do_not_change_the_referenced_columns_are_dropped(sql, keep, want):
|
||||
p = project_select(sql, keep, "sqlite")
|
||||
assert p.refused is None and p.sql == want
|
||||
assert p.kept == tuple(keep) and p.dropped == len(output_names(sql, "sqlite")) - len(keep)
|
||||
|
||||
|
||||
def test_the_text_around_the_select_list_is_kept_as_written():
|
||||
sql = "SELECT\n SUM(amount) AS sales, -- 金额\n COUNT(*) AS n\nFROM orders o\nWHERE o.status = 'paid' -- 只算已支付"
|
||||
p = project_select(sql, [0], "sqlite")
|
||||
assert p.sql == "SELECT\n SUM(amount) AS sales\nFROM orders o\nWHERE o.status = 'paid' -- 只算已支付"
|
||||
assert p.names == ("sales",)
|
||||
|
||||
|
||||
def test_keeping_everything_in_order_gives_the_sql_back_unchanged():
|
||||
sql = "SELECT SUM(amount) AS s FROM orders"
|
||||
p = project_select(sql, [0], "sqlite", width=1)
|
||||
assert p.sql == sql and p.dropped == 0 and p.refused is None
|
||||
|
||||
|
||||
def test_template_projection_keeps_the_slots():
|
||||
template = ("SELECT SUM(amount) AS gmv, COUNT(*) AS n, '{{ input.start }}' AS since FROM orders "
|
||||
"WHERE status = 'paid' AND paid_at >= '{{ input.start }}' AND paid_at < '{{ input.end }}'")
|
||||
assert output_names(template, "sqlite", template=True) == ["gmv", "n", "since"]
|
||||
p = project_template(template, [0], "sqlite")
|
||||
assert p.refused is None and p.sql == WEEKLY and p.dropped == 2 and p.names == ("gmv",)
|
||||
p = project_template(template, [1], "sqlite")
|
||||
assert p.sql == WEEKLY.replace("SUM(amount) AS gmv", "COUNT(*) AS n")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 去了会变的一律不去
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, keep, dialect", [
|
||||
("SELECT DISTINCT region, amount FROM orders", [1], "sqlite"),
|
||||
("SELECT DISTINCT ON (region) region, amount FROM orders ORDER BY region, amount", [1], "postgres"),
|
||||
# 多一个分组列：它在 GROUP BY 里
|
||||
("SELECT region, SUM(amount) AS s FROM orders GROUP BY region", [1], "sqlite"),
|
||||
("SELECT region AS r, SUM(amount) AS s FROM orders GROUP BY 1", [1], "sqlite"),
|
||||
# 按别的列排序再取前几行
|
||||
("SELECT amount, paid_at FROM orders ORDER BY paid_at DESC LIMIT 5", [0], "sqlite"),
|
||||
("SELECT amount, id FROM orders ORDER BY amount LIMIT 3 OFFSET 1", [0], "sqlite"),
|
||||
# HAVING 用到多出来的聚合：按别名、按同样的写法
|
||||
("SELECT region, SUM(amount) AS s, COUNT(*) AS n FROM orders GROUP BY region HAVING n > 2", [0, 1], "sqlite"),
|
||||
("SELECT region, SUM(amount) AS s, COUNT(*) AS n FROM orders GROUP BY region HAVING COUNT(*) > 2", [0, 1],
|
||||
"sqlite"),
|
||||
# ORDER BY 用到、按位置引用
|
||||
("SELECT region, SUM(amount) AS s, COUNT(*) AS n FROM orders GROUP BY region ORDER BY n DESC", [0, 1], "sqlite"),
|
||||
("SELECT region, SUM(amount) AS s, COUNT(*) AS n FROM orders GROUP BY region ORDER BY 3 DESC", [0, 1], "sqlite"),
|
||||
# 窗口、QUALIFY 用到
|
||||
("SELECT id, amount, SUM(amount) OVER (ORDER BY id) AS run FROM orders", [0, 2], "sqlite"),
|
||||
("SELECT region, amount, ROW_NUMBER() OVER (PARTITION BY region ORDER BY amount) AS rn FROM orders QUALIFY rn = 1",
|
||||
[0, 1], "duckdb"),
|
||||
# SQLite 的 WHERE 认列别名
|
||||
("SELECT amount, DATE(paid_at) AS d FROM orders WHERE d >= '2026-10-05'", [0], "sqlite"),
|
||||
# 集合运算、*
|
||||
("SELECT SUM(amount) AS s, COUNT(*) AS n FROM orders UNION ALL SELECT 1, 2", [0], "sqlite"),
|
||||
("SELECT *, amount * 2 AS twice FROM orders", [1], "sqlite"),
|
||||
# 可能把一行展开成多行的函数、不认识的函数
|
||||
("SELECT id, unnest(tags) AS tag FROM orders", [0], "postgres"),
|
||||
("SELECT id, generate_series(1, 3) AS k FROM orders", [0], "postgres"),
|
||||
("SELECT id, json_array_elements(payload) AS e FROM orders", [0], "postgres"),
|
||||
# 去掉仅有的聚合：剩下的不再只是一行
|
||||
("SELECT COUNT(*) AS n, '2026' AS y FROM orders", [1], "sqlite"),
|
||||
# 留下按任意一行取值的列（SQLite 取 MAX 那一行）
|
||||
("SELECT region, MAX(amount) AS m, COUNT(*) AS n FROM orders", [0, 1], "sqlite"),
|
||||
# MySQL 里紧跟着字的 -- 是两个减号：改写之后位置对不上原文
|
||||
("SELECT SUM(amount)--1 AS s, COUNT(*) AS n FROM orders", [0], "mysql"),
|
||||
], ids=["distinct", "distinct-on", "group-by-column", "group-by-ordinal", "order-limit", "order-limit-offset",
|
||||
"having-alias", "having-same-aggregate", "order-by-alias", "order-by-ordinal", "window", "qualify",
|
||||
"sqlite-where-alias", "union", "star", "unnest", "generate-series", "unknown-function", "only-aggregate",
|
||||
"loose-column", "mysql-dashes"])
|
||||
def test_items_whose_removal_could_change_the_rest_are_never_dropped(sql, keep, dialect):
|
||||
p = project_select(sql, keep, dialect)
|
||||
assert p.sql is None and p.refused, (sql, p)
|
||||
|
||||
|
||||
def test_the_result_width_must_match_the_select_list():
|
||||
p = project_select("SELECT SUM(amount) AS s, COUNT(*) AS n FROM orders", [0], "sqlite", width=3)
|
||||
assert p.sql is None and "列数" in (p.refused or "")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("keep", [[], [0, 0], [5]])
|
||||
def test_bad_keep_lists_are_refused(keep):
|
||||
assert project_select("SELECT SUM(amount) AS s, COUNT(*) AS n FROM orders", keep, "sqlite").sql is None
|
||||
|
||||
|
||||
def test_unreorderable_items_keep_their_order():
|
||||
# DuckDB 认前面的列别名：调换之后 b 就找不到 a 了
|
||||
p = project_select("SELECT amount + 1 AS a, a * 2 AS b FROM orders", [1, 0], "duckdb")
|
||||
assert p.sql is None and p.refused
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 执行守门：投影得了的，结果就是原结果的那几列
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def databases():
|
||||
conns = sql_pairs.build_databases()
|
||||
yield conns
|
||||
for c in conns:
|
||||
c.close()
|
||||
|
||||
|
||||
def _keeps(width: int) -> list[tuple[int, ...]]:
|
||||
"""每一列单独留下，再加上「只去掉一列」的几种（列多的只试前四列）。"""
|
||||
singles = [(i,) for i in range(min(width, 4))]
|
||||
drops = [tuple(j for j in range(width) if j != i) for i in range(min(width, 4))] if width > 2 else []
|
||||
return singles + drops
|
||||
|
||||
|
||||
def _bag(rows: list[tuple]) -> list[tuple]:
|
||||
"""同 sql_pairs.run：行按值排序（先后不算结果的一部分），浮点数按 6 位小数取整。"""
|
||||
rows = [tuple(sql_pairs._norm(v) for v in row) for row in rows]
|
||||
return sorted(rows, key=lambda r: tuple((v is None, str(type(v)), v if v is not None else 0) for v in r))
|
||||
|
||||
|
||||
def _corpus() -> list[str]:
|
||||
seen: dict[str, None] = {}
|
||||
for pair in sql_pairs.PAIRS:
|
||||
seen.setdefault(pair.a)
|
||||
seen.setdefault(pair.b)
|
||||
return list(seen)
|
||||
|
||||
|
||||
def test_projection_keeps_the_values_of_the_kept_columns(databases):
|
||||
"""语料里每一条 SQL：投影得了的，在每组数据上投影后的结果（行的先后不算）都等于原结果只取留下的那几列。同时数一数
|
||||
投影了多少种，免得规则收得太紧、守门形同虚设。"""
|
||||
projected = 0
|
||||
for sql in _corpus():
|
||||
width = len(output_names(sql, "sqlite") or [])
|
||||
if width < 2:
|
||||
continue
|
||||
for keep in _keeps(width):
|
||||
p = project_select(sql, keep, "sqlite", width=width)
|
||||
if p.sql is None:
|
||||
continue
|
||||
projected += 1
|
||||
for conn in databases:
|
||||
try:
|
||||
full = conn.execute(sql).fetchall()
|
||||
except Exception: # noqa: BLE001 - 语料里有只在别的方言里成立的写法：原句跑不了的不比
|
||||
break
|
||||
want = _bag([tuple(row[k] for k in keep) for row in full])
|
||||
assert sql_pairs.run(conn, p.sql) == want, (sql, keep, p.sql)
|
||||
assert projected >= 100, projected
|
||||
@@ -436,13 +436,20 @@ export function inputSource(input: EvidenceInput): { tone: SourceTone; text: str
|
||||
? { tone: 'warn', text: EVIDENCE_TEXT.codeSource(input.node_id ?? '') }
|
||||
: { tone: 'alert', text: EVIDENCE_TEXT.codeCompute(input.node_id ?? '') }
|
||||
}
|
||||
if (input.via === 'metric' && input.upstream) {
|
||||
// 转抄另一张口径卡的指标：出处跨卡追下去了（服务端记在 upstream 上），追没追到照实说
|
||||
return input.upstream.traced === true
|
||||
? { tone: 'ok', text: EVIDENCE_TEXT.upstreamTraced }
|
||||
: { tone: 'warn', text: EVIDENCE_TEXT.upstreamStop }
|
||||
}
|
||||
if (status === 'missing' || input.value == null) return { tone: 'warn', text: EVIDENCE_TEXT.inputMissing }
|
||||
return { tone: 'muted', text: '' }
|
||||
}
|
||||
|
||||
/** 值得在「输入来源」里单独列一行的输入：能核对到快照的、代码节点的、缺的 */
|
||||
/** 值得在「输入来源」里单独列一行的输入：能核对到快照的、代码节点的、转抄另一张口径卡的指标（跨卡追下去的）、缺的 */
|
||||
export const notableInput = (input: EvidenceInput): boolean =>
|
||||
input.via === 'agent_field' || input.via === 'tool_cell' || input.via === 'code'
|
||||
|| (input.via === 'metric' && !!input.upstream)
|
||||
|| input.status === 'missing' || input.value == null
|
||||
|
||||
/**
|
||||
|
||||
@@ -313,6 +313,15 @@ export const EVIDENCE_TEXT = {
|
||||
originChipHint: '这个数由口径卡确定性算出、复算一致，但算式用到的输入没有一路追到查询快照',
|
||||
/** 按位置取的格穿过了原样解析查询结果的整形节点 */
|
||||
through: (label: string) => `经「${label}」原样解析`,
|
||||
/**
|
||||
* 转抄另一张口径卡的指标（对账卡、召回卡的「口径」那一项）：出处跨卡往下追，接上那个指标自己的一段，一直到查询格子
|
||||
*/
|
||||
upstream: '转抄的口径卡指标',
|
||||
upstreamFrom: (card: string, metric: string) => `取自口径卡「${card}」的指标「${metric}」`,
|
||||
gotoUpstream: (card: string) => `查看口径卡「${card}」的这个指标`,
|
||||
upstreamTraced: '取自另一张口径卡的指标，那个指标的输入都追到了原始数据',
|
||||
upstreamStop: '取自另一张口径卡的指标，那个指标的输入没有都追到原始数据',
|
||||
upstreamUnreadable: '读不到那张口径卡的指标集（文件缺失或与哈希不一致），这一步不能作为证据',
|
||||
input: '运行输入',
|
||||
seal: '封存',
|
||||
sealOk: '已封存 · 核对一致',
|
||||
|
||||
@@ -365,7 +365,25 @@ function SegmentBody({ panelId, doc, artifact, runId, runClass, seg, unit, block
|
||||
head.scrollIntoView({ block: 'nearest' })
|
||||
flashOnce(part)
|
||||
}, [panelId])
|
||||
const linkOf = (inp: EvidenceInput) => {
|
||||
// 转抄另一张口径卡的指标（对账卡、召回卡的「口径」那一项）：服务端跨卡接上的那几步，输入点一下跳到对应的那一步
|
||||
const upstreams = chain.filter((s) => s.step === 'upstream')
|
||||
const upstreamId = (i: number) => `${panelId}-u${i}`
|
||||
const jumpToUpstream = useCallback((i: number) => {
|
||||
const part = document.getElementById(`${panelId}-u${i}`)
|
||||
const head = part?.querySelector<HTMLElement>('[data-ev-upstream-head]')
|
||||
if (!part || !head) return
|
||||
head.focus({ preventScroll: true })
|
||||
head.scrollIntoView({ block: 'nearest' })
|
||||
flashOnce(part)
|
||||
}, [panelId])
|
||||
/** depth：这个输入在第几层（指标自己的输入是 1，转抄的那一步里的输入是那一步的 depth + 1） */
|
||||
const linkOf = (inp: EvidenceInput, depth = 1): InputLink => {
|
||||
const u = inp.via === 'metric' && inp.upstream
|
||||
? upstreams.findIndex((s) => s.of === inp.path && (s.depth ?? 1) === depth) : -1
|
||||
if (u >= 0) {
|
||||
const card = host.nodeLabel?.(upstreams[u].node_id ?? '') || upstreams[u].label || upstreams[u].node_id || ''
|
||||
return { index: u, alias: card, text: EVIDENCE_TEXT.gotoUpstream(card), go: () => jumpToUpstream(u) }
|
||||
}
|
||||
const i = queryOf(inp, queries)
|
||||
return i >= 0 ? { index: i, alias: queries[i].alias ?? '', go: () => jumpTo(i) } : undefined
|
||||
}
|
||||
@@ -492,9 +510,14 @@ function SegmentBody({ panelId, doc, artifact, runId, runClass, seg, unit, block
|
||||
)}
|
||||
|
||||
{isMetric && inputs.some(notableInput) && (
|
||||
<SourcesPart inputs={inputs.filter(notableInput)} linkOf={linkOf} />
|
||||
<SourcesPart inputs={inputs.filter(notableInput)} linkOf={(inp) => linkOf(inp)} />
|
||||
)}
|
||||
|
||||
{isMetric && upstreams.map((u, i) => (
|
||||
<UpstreamPart key={`${u.artifact ?? ''}-${u.metric ?? ''}-${i}`} id={upstreamId(i)} step={u} seal={seal}
|
||||
linkOf={(inp) => linkOf(inp, (u.depth ?? 1) + 1)} />
|
||||
))}
|
||||
|
||||
{isInput && !isMetric && (
|
||||
<Part title={EVIDENCE_TEXT.input} data-ev-input-step="">
|
||||
<p className="mono">
|
||||
@@ -983,7 +1006,8 @@ function askError(error: unknown): string {
|
||||
return JUDGE_TEXT.failed(humanizeError(error).title)
|
||||
}
|
||||
|
||||
type InputLink = { index: number; alias: string; go: () => void } | undefined
|
||||
/** 输入点一下跳到哪：查询步骤（alias 是查询编号），或者转抄的那一步（text 是按钮上的字，alias 是那张卡的名字） */
|
||||
type InputLink = { index: number; alias: string; go: () => void; text?: string } | undefined
|
||||
|
||||
/**
|
||||
* 面板里提到的节点。画布右栏里是按钮：点一下在画布上选中并对准它（画布上已经没有这个节点
|
||||
@@ -1557,8 +1581,8 @@ function InputChip({ input, link }: { input: EvidenceInput; link?: InputLink })
|
||||
{link ? (
|
||||
<button type="button" className="chip mono tnum max-w-full transition-colors hover:bg-hover" style={style}
|
||||
data-ev-input={input.path ?? ''} data-ev-input-masked={masked ? '' : undefined}
|
||||
title={`${from ? `${from} · ` : ''}${EVIDENCE_TEXT.gotoQuery(link.alias)}`}
|
||||
aria-label={`${shown} ← ${input.path ?? NONE}，${EVIDENCE_TEXT.gotoQuery(link.alias)}`}
|
||||
title={`${from ? `${from} · ` : ''}${link.text ?? EVIDENCE_TEXT.gotoQuery(link.alias)}`}
|
||||
aria-label={`${shown} ← ${input.path ?? NONE}，${link.text ?? EVIDENCE_TEXT.gotoQuery(link.alias)}`}
|
||||
onClick={link.go}>
|
||||
{body}<CornerDownRight size={9} className="shrink-0 opacity-60" aria-hidden />
|
||||
</button>
|
||||
@@ -1591,6 +1615,86 @@ const TONE: Record<SourceTone, { color: string; border: string; background?: str
|
||||
muted: { color: 'var(--text-dim)', border: 'var(--border)' },
|
||||
}
|
||||
|
||||
/**
|
||||
* 转抄的口径卡指标（对账卡、召回卡的「口径」那一项）：服务端跨卡接上的一步——那张卡的那个指标、它的值、原式和代入式，
|
||||
* 它自己的输入（点一下跳到口径的取数查询，或者再下一层转抄的那一步），追没追到原始数据。跨卡怎么追是服务端定的
|
||||
* （api/evidence._upstream_steps），这里只照着画
|
||||
*/
|
||||
function UpstreamPart({ id, step, seal, linkOf }: {
|
||||
id: string; step: EvidenceStep; seal?: EvidenceSeal; linkOf: (input: EvidenceInput) => InputLink
|
||||
}) {
|
||||
const host = useEvidenceHost()
|
||||
const card = host.nodeLabel?.(step.node_id ?? '') || step.label || step.node_id || NONE
|
||||
const inputs = (step.inputs ?? []).filter((i) => i && typeof i === 'object')
|
||||
const unreadable = step.hash_ok === false || step.name == null
|
||||
const untraced = Array.isArray(step.untraced) ? step.untraced.filter((u) => u && typeof u === 'object') : null
|
||||
const stopAt = (u: EvidenceUntraced) => (u.node_id
|
||||
? EVIDENCE_TEXT.originStop(u.path,
|
||||
EVIDENCE_TEXT.originNode(host.nodeLabel?.(u.node_id) || u.label || u.node_id, u.type ? nodeTypeLabel(u.type) : ''))
|
||||
: EVIDENCE_TEXT.originUnknown(u.path))
|
||||
return (
|
||||
<div id={id} data-ev-upstream={step.node_id ?? ''} data-ev-upstream-metric={step.metric ?? ''}
|
||||
className="rounded data-[flash]:outline data-[flash]:outline-solid data-[flash]:outline-2 data-[flash]:outline-offset-2 data-[flash=focus]:outline-[color:var(--accent)]">
|
||||
<div className="mb-1 text-2xs font-medium text-faint">{EVIDENCE_TEXT.upstream}</div>
|
||||
<div className="flex flex-wrap items-baseline gap-x-2">
|
||||
<span tabIndex={-1} data-ev-upstream-head=""
|
||||
className="rounded-sm font-medium focus:outline-none focus-visible:outline focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-offset-1 focus-visible:outline-[color:var(--accent)]">
|
||||
{EVIDENCE_TEXT.upstreamFrom(card, step.name || step.metric || NONE)}
|
||||
</span>
|
||||
{!unreadable && step.rendered && <span className="mono tnum text-sm" data-ev-upstream-value="">{step.rendered}</span>}
|
||||
{step.incomplete && (
|
||||
<span className="chip" title={step.incomplete_reason || EVIDENCE_TEXT.incompleteFallback}
|
||||
style={{ color: 'var(--st-waiting)', borderColor: 'var(--st-waiting)' }}>{EVIDENCE_TEXT.incomplete}</span>
|
||||
)}
|
||||
{step.sql_check_failed && (
|
||||
<span className="chip" title={step.sql_check_reason || EVIDENCE_TEXT.sqlCheckFallback}
|
||||
style={{ color: 'var(--st-waiting)', borderColor: 'var(--st-waiting)' }}>{EVIDENCE_TEXT.sqlCheckFailed}</span>
|
||||
)}
|
||||
{host.onNode && <NodeChip id={step.node_id} />}
|
||||
</div>
|
||||
<Integrity step={step} seal={seal} />
|
||||
{unreadable ? (
|
||||
<p className="mt-1" style={{ color: 'var(--st-failed)' }} data-ev-upstream-unreadable="">
|
||||
{EVIDENCE_TEXT.upstreamUnreadable}
|
||||
</p>
|
||||
) : (
|
||||
<dl className="mt-2 space-y-1.5">
|
||||
<div>
|
||||
<dt className="text-2xs text-faint">{EVIDENCE_TEXT.expression}</dt>
|
||||
<dd><code className="mono block whitespace-pre-wrap break-all rounded bg-bg px-1.5 py-1 text-2xs">
|
||||
{step.expression || NONE}
|
||||
</code></dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt className="text-2xs text-faint">{EVIDENCE_TEXT.substituted}</dt>
|
||||
<dd>
|
||||
<code className="mono block whitespace-pre-wrap break-all rounded bg-bg px-1.5 py-1 text-2xs">
|
||||
{step.substituted || NONE}
|
||||
</code>
|
||||
{!!inputs.length && (
|
||||
<ul className="mt-1 flex flex-wrap gap-1" aria-label={EVIDENCE_TEXT.inputs} data-ev-upstream-inputs="">
|
||||
{inputs.map((inp, i) => <InputChip key={`${inp.path}-${i}`} input={inp} link={linkOf(inp)} />)}
|
||||
</ul>
|
||||
)}
|
||||
</dd>
|
||||
</div>
|
||||
{untraced && (
|
||||
<div>
|
||||
<dt className="text-2xs text-faint">{EVIDENCE_TEXT.origin}</dt>
|
||||
<dd data-ev-upstream-origin={untraced.length ? 'untraced' : 'traced'}
|
||||
style={{ color: untraced.length ? 'var(--st-waiting)' : 'var(--st-done)' }}>
|
||||
{untraced.length ? (
|
||||
<ul className="space-y-0.5">{untraced.map((u, i) => <li key={`${u.path}-${i}`}>{stopAt(u)}</li>)}</ul>
|
||||
) : EVIDENCE_TEXT.originTraced}
|
||||
</dd>
|
||||
</div>
|
||||
)}
|
||||
</dl>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 输入来源：每个能核对到快照的输入（agent 字段、cell() 取数）、代码节点的输入、缺的输入各一行——
|
||||
* 路径、从哪个节点哪一格来、核对结果。能对到查询步骤的给「看查询 Qn」
|
||||
@@ -1606,7 +1710,8 @@ function SourcesPart({ inputs, linkOf }: { inputs: EvidenceInput[]; linkOf: (inp
|
||||
const link = linkOf(inp)
|
||||
// 查询编号写报告目录里的全局编号（接口补的 query、对上的查询步骤）；agent 字段自己的 ref 是
|
||||
// 那个节点内部的编号，和面板里的「查询 Qn」不是一回事，不拿来写
|
||||
const alias = link?.alias || inp.query || (typeof inp.cell === 'string' ? inp.cell.split('.')[0] : '')
|
||||
const alias = (link && !link.text ? link.alias : '') || inp.query
|
||||
|| (typeof inp.cell === 'string' ? inp.cell.split('.')[0] : '')
|
||||
const where = [
|
||||
inp.field && (inp.via === 'agent_field' ? `字段 ${inp.field}` : inp.field),
|
||||
throughText(inp, host.nodeLabel),
|
||||
@@ -1621,7 +1726,7 @@ function SourcesPart({ inputs, linkOf }: { inputs: EvidenceInput[]; linkOf: (inp
|
||||
<span className="mono min-w-0 flex-1 truncate" title={inp.path}>{inp.path ?? NONE}</span>
|
||||
{link && (
|
||||
<button type="button" className="btn btn-xs btn-ghost -mr-1 shrink-0" onClick={link.go} data-ev-goto="">
|
||||
<CornerDownRight size={10} aria-hidden /> {EVIDENCE_TEXT.gotoQuery(link.alias)}
|
||||
<CornerDownRight size={10} aria-hidden /> {link.text ?? EVIDENCE_TEXT.gotoQuery(link.alias)}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -2171,6 +2171,19 @@ export interface EvidenceInput {
|
||||
* 不进 masked：面板底下「这些列设置了遮罩」不把它们算进去
|
||||
*/
|
||||
mask_unverified?: string
|
||||
/**
|
||||
* via 为 metric、转抄另一张口径卡的指标（nodes.<卡>.metrics[i].value）时，运行时跨卡往下追的出处：那个指标的 id、那张卡的
|
||||
* 指标集工件、转抄了几层、那个指标的输入是不是都追到了原始数据（服务端 engine/lineage.metric_copy）。老运行、别的输入没有
|
||||
*/
|
||||
upstream?: EvidenceUpstreamRef | null
|
||||
}
|
||||
|
||||
/** 转抄的输入跨卡往下追的出处（EvidenceInput.upstream） */
|
||||
export interface EvidenceUpstreamRef {
|
||||
metric?: string
|
||||
artifact?: string | null
|
||||
depth?: number
|
||||
traced?: boolean
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -2319,6 +2332,14 @@ export interface EvidenceUntraced {
|
||||
|
||||
export interface EvidenceStep extends Omit<EvidenceInput, 'status'> {
|
||||
step: string
|
||||
/**
|
||||
* upstream：转抄另一张口径卡的指标时跨卡接上的那一步（api/evidence._upstream_steps）。of 是接在哪个输入后面（输入的路径），
|
||||
* label 是那张卡在图里的名字，traced 是那个指标的输入都追到了原始数据，depth 是转抄了几层
|
||||
*/
|
||||
of?: string
|
||||
label?: string | null
|
||||
traced?: boolean
|
||||
depth?: number
|
||||
metric?: string
|
||||
decimals?: number | null
|
||||
format?: string
|
||||
|
||||
@@ -8,6 +8,8 @@
|
||||
// - 口径对账：核对参数（推断的参数要先核对）→ 发起（认不出列怎么配时照 409 带回的选项选对照的指标，再发起）→ 结果卡片：
|
||||
// 这次 / 口径 / 差值 / 相对差四个数各自取值、点开是出处链；请求体逐项核对；做完的卡片上问「差在哪」的说明有没有帮助
|
||||
// （说明有用 / 看不懂，统计 E2）：请求体只有 verdict，选过的按下、写各几个人，能改主意；没做完的不问；
|
||||
// - 「按口径」那个数转抄口径卡的指标：出处链跨卡接上口径卡那个指标的一步，输入来源里点一下跳到那一步，那一步的输入
|
||||
// 点一下跳到口径的取数查询；
|
||||
// - 可沉淀的问题：工作流列表弹窗里的抽屉，三类来源分开写；「固化成口径草稿」走 from-run 的 freeze（带同组的别的运行）、
|
||||
// 打开画布；聚类按人挑的那一轮固化；「不是口径」要写理由；
|
||||
// - 使用者看到的入口照服务端的动作：不能出具的写原因（请搭建者出具）、不摆出具按钮，能对账的照样对账；出具过的正式运行
|
||||
@@ -22,7 +24,7 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { chromium } from '../frontend/node_modules/playwright-core/index.mjs'
|
||||
import {
|
||||
ISSUE, QUEUE, RECONCILE, RECONCILE_VALUES, WHOLE, WHOLE_LABELS, ledgerOf, metricDetail, reconciledOf,
|
||||
ISSUE, QUEUE, RECONCILE, RECONCILE_VALUES, WHOLE, WHOLE_LABELS, copiedMetricDetail, ledgerOf, metricDetail, reconciledOf,
|
||||
} from './ledger-fixtures.mjs'
|
||||
|
||||
const WEB = process.env.AGENTLAB_WEB ?? 'http://localhost:5273'
|
||||
@@ -278,6 +280,52 @@ await section('口径对账：核对参数 → 发起 → 选对照的指标 →
|
||||
check('没有运行时报错', errors.length === 0, errors.join(' | '))
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------- 「按口径」那个数的出处跨卡追到取数查询
|
||||
|
||||
await section('对账结果里「按口径」那个数：出处跨卡追到口径的取数查询', async () => {
|
||||
const extra = (method, p) => {
|
||||
const m = p.match(/^\/runs\/run-rec-1\/evidence\/metrics\/reconcile\/([^/]+)$/)
|
||||
if (method === 'GET' && m) {
|
||||
return { json: m[1] === 'cal_0' ? copiedMetricDetail('run-rec-1', m[1]) : metricDetail('run-rec-1', m[1]) }
|
||||
}
|
||||
return null
|
||||
}
|
||||
const { page, errors } = await open('/ui-harness.html?evidence=1', {
|
||||
reply: evidenceReply(() => ledgerOf(RUN, { reconciled: [reconciledOf('succeeded')] }), extra) })
|
||||
await page.waitForSelector('#evidence-wide [data-seg="s8"]', { timeout: 10000 })
|
||||
await openSeg(page, 's8')
|
||||
const card = page.locator('[data-evidence-panel] [data-ledger-reconciled="run-rec-1"]')
|
||||
await card.waitFor({ timeout: 6000 })
|
||||
await until(async () => (await card.innerText()).includes(RECONCILE_VALUES.cal_0), 5000)
|
||||
await card.locator('[data-ledger-value="cal_0"]').click()
|
||||
const drawer = page.locator('[data-evidence-panel="side"]').last()
|
||||
const up = drawer.locator('[data-ev-upstream="card"]')
|
||||
const shown = await up.waitFor({ timeout: 5000 }).then(() => true, () => false)
|
||||
const upText = shown ? (await up.innerText()).replace(/\s+/g, ' ') : ''
|
||||
check('点「按口径」：出处链接上口径卡的那个指标（转抄的口径卡指标）', shown && upText.includes('转抄的口径卡指标')
|
||||
&& upText.includes('取自口径卡「口径卡」的指标「订单数」') && upText.includes('1,180单'), upText)
|
||||
check('……写那个指标的原式、它的输入追到了原始数据', upText.includes("cell(nodes.fetch, 0, 'orders')")
|
||||
&& await up.locator('[data-ev-upstream-origin="traced"]').count() === 1, upText)
|
||||
check('指标这一步：输入都追到了原始数据，不挂「未追到原始数据」', await drawer.locator('[data-ev-untraced-chip]').count() === 0
|
||||
&& await drawer.locator('[data-ev-metric] [data-ev-origin="traced"]').count() === 1)
|
||||
const src = drawer.locator('[data-ev-source="nodes.card.metrics[0].value"]')
|
||||
const srcText = await src.innerText().catch(() => '')
|
||||
check('输入来源：转抄的那一项说清取自另一张口径卡、那个指标的输入都追到了原始数据',
|
||||
srcText.includes('取自另一张口径卡的指标，那个指标的输入都追到了原始数据')
|
||||
&& srcText.includes('查看口径卡「口径卡」的这个指标'), srcText)
|
||||
await src.locator('[data-ev-goto]').click()
|
||||
const focused = await until(() => page.evaluate(() => document.activeElement?.hasAttribute('data-ev-upstream-head') === true), 3000)
|
||||
check('点「查看口径卡「口径卡」的这个指标」：焦点落到那一步、描一下边', focused
|
||||
&& await up.getAttribute('data-flash') === 'focus')
|
||||
await up.locator(`[data-ev-input="cell(nodes.fetch, 0, 'orders')"]`).click()
|
||||
const query = drawer.locator('[data-ev-query="q-cal-fetch"]')
|
||||
const jumped = await until(async () => await query.getAttribute('data-flash') === 'focus', 3000)
|
||||
check('那个指标的输入点一下：跳到口径的取数查询（SQL、被引用的那一格都在）', jumped
|
||||
&& (await query.innerText()).includes('SELECT COUNT(*) AS orders FROM orders')
|
||||
&& await query.locator('td[data-highlight="cell"]').count() === 1)
|
||||
check('没有运行时报错', errors.length === 0, errors.join(' | '))
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------- 可沉淀的问题
|
||||
|
||||
const WF = { id: 'wf-led-check', name: '检查用工作流', description: '', version: 1, status: 'draft', tags: [], is_template: false,
|
||||
|
||||
@@ -112,6 +112,31 @@ export const metricDetail = (runId, metric) => ({
|
||||
seal: { sealed: true, ok: true, covered: true }, redacted: { columns: [] },
|
||||
})
|
||||
|
||||
/**
|
||||
* 对账卡上「按口径」那个数的出处（服务端 api/evidence._upstream_steps 的形状）：它转抄口径卡的指标
|
||||
* （nodes.card.metrics[0].value），出处跨卡接上口径卡「订单数」那一步，再到口径的取数查询（q-cal-fetch）
|
||||
*/
|
||||
export const copiedMetricDetail = (runId, metric) => ({
|
||||
run_id: runId, node: 'reconcile', metric, rendered: RECONCILE_VALUES[metric] ?? '—', eid: `ev:metric:${metric}`,
|
||||
chain: [
|
||||
{ step: 'metric', alias: 'm', metric, name: '订单数 · 口径', value: 1180, rendered: RECONCILE_VALUES[metric], node_id: 'reconcile',
|
||||
caliber: '口径对账', version: 'v1', expression: 'nodes.card.metrics[0].value', substituted: '1180', recompute_ok: true,
|
||||
sealed: true, hash_ok: true, untraced: [],
|
||||
inputs: [{ path: 'nodes.card.metrics[0].value', value: 1180, node_id: 'card', via: 'metric', field: 'metrics[0].value',
|
||||
status: 'ok', upstream: { metric: 'orders', artifact: 'ms-card', depth: 1, traced: true } }] },
|
||||
{ step: 'upstream', of: 'nodes.card.metrics[0].value', node_id: 'card', label: '口径卡', artifact: 'ms-card', metric: 'orders',
|
||||
traced: true, depth: 1, hash_ok: true, sealed: true, name: '订单数', value: 1180, unit: '单', rendered: '1,180单',
|
||||
caliber: '订单口径', version: 'v1', expression: "cell(nodes.fetch, 0, 'orders')", substituted: '1180', recompute_ok: true,
|
||||
status: 'ok', untraced: [],
|
||||
inputs: [{ step: 'input', path: "cell(nodes.fetch, 0, 'orders')", value: 1180, node_id: 'fetch', via: 'tool_cell',
|
||||
status: 'verified', artifact: 'q-cal-fetch', locator: { row: 0, column: 'orders' }, eid: 'ev:cell:q-cal-fetch' }] },
|
||||
{ step: 'query', alias: null, artifact: 'q-cal-fetch', node_id: 'fetch', tool: 'db_query__shop', source: 'shop',
|
||||
sql: "SELECT COUNT(*) AS orders FROM orders WHERE status = 'paid'", columns: ['orders'], rows: [[1180]], row_offset: 0,
|
||||
row_index: [0], total_rows: 1, highlight: { rows: [0], cols: ['orders'], cells: [[0, 'orders']] }, sealed: true, hash_ok: true },
|
||||
],
|
||||
seal: { sealed: true, ok: true, covered: true }, redacted: { columns: [] },
|
||||
})
|
||||
|
||||
/** 认证队列：三类来源各一项 */
|
||||
export const QUEUE = {
|
||||
items: [
|
||||
|
||||
Reference in New Issue
Block a user