合并查询节点(二):证据面板展示合并的输入、合并 SQL 和警告,被引用的格沿逐格来历追到输入的那一格并接上输入的查询步骤;推断来源穿过合并下钻到原表格子,追不到的判为没有逐格来历;裁判摘录写明合并的输入并按底层数据源遮罩

This commit is contained in:
ThearchyHelios
2026-10-03 09:36:20 +08:00
parent 1f8c180703
commit 8f75849a1c
8 changed files with 460 additions and 44 deletions
+194 -11
View File
@@ -58,7 +58,9 @@ from app.data.provenance_types import (
CellRef,
Chain,
ChainProblem,
REASON_TEXT,
LocateProblem,
MergeHop,
ProvenanceOut,
RawState,
Reason,
@@ -100,6 +102,7 @@ from app.engine.evidence import (
uncited_claims,
)
from app.engine.judge import FIELDS_MAX, SETTLED, UNSUPPORTED, VERDICTS, candidates, outdated
from app.engine.merge_query import MERGE_SOURCE, trace_cell
from app.engine.toolcalls import QUERY_PREFIX
router = APIRouter(prefix="/api/runs", tags=["evidence"])
@@ -426,6 +429,12 @@ def _graph_of(report: _Report, sealed: _Sealed, inputs: dict[str, Any]) -> tuple
if entry.get("kind") in ("query", "retrieval"):
# 查询、检索条目带上是什么、有多大:图上不点开也看得出这是哪次取数
item.update({k: entry[k] for k in ("tool", "source", "columns", "rows", "truncated") if k in entry})
if entry.get("kind") == "query" and entry.get("source") == MERGE_SOURCE:
# 合并结果连向它合并的那几次查询:图上看得出 Q3 是 Q1、Q2 按键合并出来的
edges.extend({"from": alias, "to": target, "rel": "merged_from", "input": source.get("alias"),
"report": report.node_id}
for source in _merge_inputs(entry.get("artifact"))
if (target := _query_alias(catalog, source.get("artifact"))))
elif entry.get("kind") in ENTITY_KINDS:
# 表和字段:叫什么、出现在哪几次查询里、来历(表结构快照 / SQL / 结果列)各自封存了没有
item.update({k: entry[k] for k in ("name", "table", "tables", "qualified", "source") if entry.get(k)})
@@ -454,6 +463,16 @@ def _graph_of(report: _Report, sealed: _Sealed, inputs: dict[str, Any]) -> tuple
return evidence, edges
def _merge_inputs(artifact: Any) -> list[dict[str, Any]]:
"""合并结果快照记下的输入;取不回、哈希不符的当没有(证据图只是少几条边,片段接口照样说清楚)。"""
try:
snap = load(str(artifact)) if artifact else None
except ValueError:
return []
inputs = snap.get("inputs") if isinstance(snap, dict) else None
return [i for i in inputs if isinstance(i, dict)] if isinstance(inputs, list) else []
def _cell_name(alias: Any, locator: dict[str, Any]) -> str:
"""Q1.r0.gmv:报告目录里的全局编号加行列,和 [[v:]] 的写法一致。"""
if not alias or not isinstance(locator.get("row"), int) or not locator.get("column"):
@@ -778,8 +797,8 @@ async def _chain(seg: dict[str, Any], doc: dict[str, Any], sealed: _Sealed,
locator = cite.get("locator") or {}
# 只有直接引用的单元格才可能推断来源(P4-SPEC 2.8.2)。指标链里的查询步骤不带:指标片段的推断来源只会
# 回「不是单元格」,带上提示等于让界面白发一次请求
return [await _query_step(str(entry.get("artifact") or ""), [(locator.get("row"), locator.get("column"))],
doc, sealed, masks, hint=doc.get("provenance") == DOC_PROVENANCE)]
return await _query_steps(str(entry.get("artifact") or ""), [(locator.get("row"), locator.get("column"))],
doc, sealed, masks, hint=doc.get("provenance") == DOC_PROVENANCE)
if cite.get("kind") == "input":
name = (cite.get("locator") or {}).get("field")
inputs = _input_payloads(sealed)
@@ -843,7 +862,7 @@ async def _metric_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[s
one["cell"] = cell
wanted.setdefault(str(artifact_of), []).extend(_located(item.get("locator") or {}))
inputs.append(one)
queries = [await _query_step(a, cells, doc, sealed, masks) for a, cells in wanted.items()]
queries = [s for a, cells in wanted.items() for s in await _query_steps(a, cells, doc, sealed, masks)]
return [step, *inputs, *queries]
@@ -932,20 +951,30 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str
columns = [str(c) for c in snap.get("columns") or []]
rows = snap["rows"]
step["source"] = step["source"] or snap.get("source")
# 遮的是「查询当时记下的」和「数据源现在设的」两者之和:事后加的遮罩照样生效,数据源改名、
# 删掉了也不会把当时遮着的列亮出来
hidden, found = await masks.of(step["source"])
recorded = snap.get("mask_columns")
hidden = hidden | ({str(c).lower() for c in recorded if isinstance(c, str)} if isinstance(recorded, list) else set())
if not found:
step["mask_note"] = SOURCE_GONE.format(source=step["source"])
merged = snap.get("source") == MERGE_SOURCE
if merged:
# 合并查询不是数据源:遮罩按它合并的那几个数据源算(_merge_masks),不能拿「合并查询」去找数据源,
# 找不到再说它「已不存在」
hidden, gone = await _merge_masks(snap, masks)
if gone:
step["mask_note"] = SOURCE_GONE.format(source="、".join(gone))
step["merge"] = _merge_view(snap, cells, catalog, sealed)
if hint and _leads_to_upload(snap, cells):
step["provenance"] = True
else:
# 遮的是「查询当时记下的」和「数据源现在设的」两者之和:事后加的遮罩照样生效,数据源改名、
# 删掉了也不会把当时遮着的列亮出来
hidden, found = await masks.of(step["source"])
hidden = hidden | _recorded_masks(snap)
if not found:
step["mask_note"] = SOURCE_GONE.format(source=step["source"])
masked = {i for i, c in enumerate(columns) if c.lower() in hidden}
index = sorted({i for r in rows_hit for i in range(r - WINDOW, r + WINDOW + 1) if 0 <= i < len(rows)})
step.update(sql=snap.get("sql"), columns=columns, total_rows=len(rows), truncated=bool(snap.get("truncated")),
masked=[columns[i] for i in sorted(masked)],
# 查询时从驱动的原始值记下的列类型(老快照没有):文本列里的 "2026" 按文本显示,不当数
column_types=snap.get("column_types") if isinstance(snap.get("column_types"), dict) else {},
**({"provenance": True} if hint and snap.get("data_version") else {}))
**({"provenance": True} if hint and snap.get("data_version") and not merged else {}))
if len(index) > MAX_WINDOW_ROWS:
index, step["window_truncated"] = index[:MAX_WINDOW_ROWS], True
step["row_index"] = index
@@ -955,6 +984,116 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str
return step
# --------------------------------------------------------------------------
# 合并查询的结果:表级来历(输入、合并 SQL)和逐格来历(被引用的格追到输入的哪一格)
# --------------------------------------------------------------------------
#: 一条链最多追几层合并(合并的合并)。画布上不会无限嵌套,这只是防御坏快照
MERGE_DEPTH = 4
async def _query_steps(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str, Any], sealed: _Sealed,
masks: _Masks, *, hint: bool = False, depth: int = 0) -> list[dict[str, Any]]:
"""查询步骤;快照是合并结果时,后面接上被引用的格追到的那几个输入的查询步骤(高亮追到的那一格)。
追不到的格不接任何输入步骤:合并步骤的 merge.traced 里照实写「没有逐格来历」,只给表级来历。输入步骤带
merged_into(合并结果的编号),界面据此标明它是哪次合并的输入。不是合并结果的快照,返回的就是原来那一个步骤。
"""
step = await _query_step(artifact, cells, doc, sealed, masks, hint=hint)
traced = [t for t in (step.get("merge") or {}).get("traced") or [] if t.get("input")]
if not traced or depth >= MERGE_DEPTH:
return [step]
by_input: dict[str, list[tuple[Any, Any]]] = {}
artifacts = {str(i.get("alias")): str(i.get("artifact") or "") for i in step["merge"]["inputs"]}
for t in traced:
by_input.setdefault(artifacts.get(str(t["input"]), ""), []).append((t["row"], t["column"]))
out = [step]
for source, picked in by_input.items():
if not source:
continue
for sub in await _query_steps(source, picked, doc, sealed, masks, depth=depth + 1):
out.append({**sub, "merged_into": sub.get("merged_into") or step.get("alias")})
return out
def _merge_view(snap: dict[str, Any], cells: list[tuple[Any, Any]], catalog: dict[str, Any],
sealed: _Sealed) -> dict[str, Any]:
"""合并步骤的 merge 一节:每个输入(别名、节点、节点名、目录编号、行数、数据源、封存与否)、合并 SQL、执行时的
警告,以及被引用的每一格追到了哪个输入的哪一格(追不到的写明原因)。"""
labels = {str(n.get("id")): (n.get("data") or {}).get("label") or None
for n in sealed.graph.get("nodes") or [] if isinstance(n, dict)}
inputs = []
for item in snap.get("inputs") or []:
if not isinstance(item, dict):
continue
art = str(item.get("artifact") or "")
inputs.append({"alias": item.get("alias"), "node_id": item.get("node_id"),
"label": labels.get(str(item.get("node_id"))), "query": _query_alias(catalog, art),
"rows": item.get("rows"), "source": item.get("source"), "artifact": art or None,
"sealed": sealed.trusted and art in sealed.queries})
traced = []
for row, column in dict.fromkeys((r, c) for r, c in cells if isinstance(r, int) and c):
hit = trace_cell(snap, row, column)
if hit is None:
traced.append({"cell": [row, str(column)], "input": None, "query": None, "row": None, "column": None,
"note": REASON_TEXT["merge_no_lineage"]})
else:
traced.append({"cell": [row, str(column)], "input": hit.alias, "query": _query_alias(catalog, hit.artifact),
"row": hit.row, "column": hit.column})
warnings = [{"code": w.get("code"), "message": w.get("message")} for w in snap.get("warnings") or []
if isinstance(w, dict) and w.get("message")]
return {"sql": snap.get("sql"), "inputs": inputs, "warnings": warnings, "traced": traced}
async def _merge_masks(snap: dict[str, Any], masks: _Masks) -> tuple[set[str], list[str]]:
"""合并结果要遮的列(小写)和已经不在的数据源:
- 合并当时记下的(快照的 mask_columns:各输入当时的遮罩列,加上来自遮罩列、换了名字的结果列);
- 它合并的每个数据源现在设的(按列名);
- 结果列有逐格来历、来自某个数据源现在遮着的列的,按来历遮(换了名字也遮住)。宁可多遮。"""
hidden = _recorded_masks(snap)
gone: list[str] = []
for name in snap.get("sources") or []:
cols, found = await masks.of(name)
hidden |= cols
if not found:
gone.append(str(name))
lineage = snap.get("lineage") if isinstance(snap.get("lineage"), dict) else {}
origins = lineage.get("sources") or []
by_alias = {str(i.get("alias")): i for i in snap.get("inputs") or [] if isinstance(i, dict)}
for name, target in zip(snap.get("columns") or [], lineage.get("columns") or []):
if not isinstance(target, dict) or not isinstance(target.get("source"), int) \
or not 0 <= target["source"] < len(origins):
continue
origin = origins[target["source"]] if isinstance(origins[target["source"]], dict) else {}
source = (by_alias.get(str(origin.get("input"))) or {}).get("source")
if source and source != MERGE_SOURCE and str(target.get("column") or "").lower() in (await masks.of(source))[0]:
hidden.add(str(name).lower())
return hidden, gone
def _leads_to_upload(snap: dict[str, Any], cells: list[tuple[Any, Any]], depth: int = 0) -> bool:
"""被引用的格经合并追下去,最后落在上传表格的查询快照上(有 data_version):界面才去请求推断来源。"""
if depth >= MERGE_DEPTH:
return False
for row, column in cells:
hit = trace_cell(snap, row, column)
if hit is None or not hit.artifact:
continue
try:
source = load(hit.artifact)
except ValueError:
continue
if not isinstance(source, dict):
continue
if source.get("source") == MERGE_SOURCE:
if _leads_to_upload(source, [(hit.row, hit.column)], depth + 1):
return True
elif source.get("data_version"):
return True
return False
# --------------------------------------------------------------------------
# GET /api/runs/{run_id}/evidence/segments/{segment_id}/provenance(期 4:推断的来源)
# --------------------------------------------------------------------------
@@ -1070,6 +1209,10 @@ async def segment_provenance(report: _Report, seg: dict[str, Any], sealed: _Seal
# R0:期 4 之前组装的文档(没有标记)一律不推断,期 4 之前的面板不变(1.4)
if doc.get("provenance") != DOC_PROVENANCE:
return none("legacy_doc")
# 合并查询结果里的格:先按逐格来历追到输入的那一格,再对那一格照常判断;追不到就只有表级来历
if (cite.get("kind") == "cell" and cite.get("status") == "resolved" and seg.get("state") == "deterministic"
and entry.get("kind") == "query" and entry.get("source") == MERGE_SOURCE):
return await _merge_provenance(report, seg, entry, sealed, masks=masks, base=base)
# R1:只对直接引用的查询单元格下钻;代码节点、Agent 字段的取数(工具名不是 db_query__ 开头)不算
if (cite.get("kind") != "cell" or cite.get("status") != "resolved" or seg.get("state") != "deterministic"
or entry.get("kind") != "query" or not str(entry.get("tool") or "").startswith(QUERY_PREFIX)):
@@ -1094,6 +1237,46 @@ async def segment_provenance(report: _Report, seg: dict[str, Any], sealed: _Seal
masks=masks, sealed=sealed)
async def _merge_provenance(report: _Report, seg: dict[str, Any], entry: dict[str, Any], sealed: _Sealed, *,
masks: _Masks, base: dict[str, Any], hops: tuple[MergeHop, ...] = ()) -> ProvenanceOut:
"""合并结果里的一格:沿合并快照记下的逐格来历(engine/merge_query.py)追到输入的那一格,换成对那一格的判断。
宁可不下钻,也不下钻错:追不到(计算、聚合出来的列,合并 SQL 超出可追溯的写法,一模一样的两行)就判
merge_no_lineage,只有表级来历(合并的输入、合并 SQL 在查询步骤的 merge 一节里)。追到的输入是数据源查询时,
结论就是直接引用那一格时的结论(上传表格照常推断原表格子,手工源 not_upload);输入本身也是合并结果时再追一跳。
cell 始终是报告引用的那一格,merge 列出经过的每一跳。
"""
def out(status: str, reason: Reason, *more: MergeHop) -> ProvenanceOut:
return ProvenanceOut(status=status, reason=reason, merge=[*hops, *more], **base) # type: ignore[arg-type]
cite = seg.get("cite") or {}
artifact = str(entry.get("artifact") or "")
snap, ok = await _Artifacts().get(artifact) if artifact in sealed.queries else (None, None)
if ok is not True or not isinstance(snap, dict):
return out("none", refuse("not_sealed"))
locator = cite.get("locator") or {}
row, column = locator.get("row"), locator.get("column")
if isinstance(column, str) and column.lower() in _recorded_masks(snap):
return out("none", refuse("masked"))
hit = trace_cell(snap, row, column)
catalog = (report.doc or {}).get("catalog") or {}
alias = _query_alias(catalog, hit.artifact) if hit is not None else None
hop = MergeHop(alias=str(cite.get("alias") or ""), node_id=entry.get("node_id"),
input=hit.alias if hit else None, query=alias, row=hit.row if hit else None,
column=hit.column if hit else None)
if hit is None or len(hops) >= MERGE_DEPTH:
return out("none", refuse("merge_no_lineage"), replace(hop, input=None, query=None, row=None, column=None))
if alias is None:
# 输入不在这份报告的目录里(不在报告的上游、没有记进台账):认不出它是封存过的查询
return out("none", refuse("not_sealed"), hop)
sub_entry = catalog[alias]
sub_seg = {**seg, "cite": {**cite, "alias": alias, "locator": {"row": hit.row, "column": hit.column}}}
if sub_entry.get("source") == MERGE_SOURCE:
return await _merge_provenance(report, sub_seg, sub_entry, sealed, masks=masks, base=base, hops=(*hops, hop))
found = await segment_provenance(report, sub_seg, sealed, masks=masks)
return replace(found, cell=base["cell"], segment=base["segment"], merge=[*hops, hop])
async def _drill(base: dict[str, Any], *, seg: dict[str, Any], entry: dict[str, Any], query: dict[str, Any],
schema: dict[str, Any], artifacts: _Artifacts, masks: _Masks, sealed: _Sealed) -> ProvenanceOut:
"""R5–R17(P4-SPEC 2.3):第一条不满足的就是结论。
+35
View File
@@ -95,6 +95,8 @@ ReasonCode = Literal[
"recheck_missing", "recheck_multiple", "recheck_mismatch",
# R17
"no_lineage",
# 合并查询的结果:这一格说不清来自哪个输入的哪一格(计算、聚合出来的列,或合并 SQL 超出可追溯的写法)
"merge_no_lineage",
]
#: 标红的提示。出现时 reason.code 与它相同,界面只画提示、不再画 reason.text(P4-SPEC 4.1、4.2 末尾)
@@ -179,6 +181,8 @@ REASON_TEXT: dict[str, str] = {
"recheck_multiple": _RECHECK_TEXT,
"recheck_mismatch": _RECHECK_TEXT,
"no_lineage": "导入清单里没有这一格的溯源记录,只给出表级来历",
"merge_no_lineage": ("这一格来自合并查询,无法确定它对应哪个输入的哪一格(计算、聚合出来的列,或合并 SQL 超出可追溯的"
"写法),没有逐格来历,只有表级来历:合并了哪几个输入、用的哪条合并 SQL"),
}
#: 标红提示的原文:与同名 reason 的原文相同(两处同时出现时界面只画提示)
@@ -506,6 +510,27 @@ class RelatedCheck:
detail: str | None = None
@dataclass(kw_only=True)
class MergeHop:
"""经过的一次合并查询(engine/merge_query.py):被引用的格在合并结果里,追到了哪个输入的哪一格。
追不到的那一跳 input、query、row、column 都是 None,结论是 merge_no_lineage。输入本身也是合并结果时再追一跳,
ProvenanceOut.merge 按从报告引用的那一份往下的顺序列出每一跳。"""
#: 合并结果在目录里的编号(Q3)
alias: str
#: 合并查询节点
node_id: str | None = None
#: 追到的输入别名(合并 SQL 里的表名)
input: str | None = None
#: 那个输入在目录里的编号(Q1);输入不在这份报告的目录里时为 None
query: str | None = None
#: 输入快照里的行号,从 0 数
row: int | None = None
#: 输入快照里的列名
column: str | None = None
@dataclass(kw_only=True)
class ProvenanceOut:
"""推断来源接口的响应。asdict 之后就是 P4-SPEC 2.8.1 的 JSON;不变式见 contract_problems。"""
@@ -523,6 +548,9 @@ class ProvenanceOut:
version: VersionView | None = None
cell_source: CellSource | None = None
checks: list[RelatedCheck] = field(default_factory=list)
#: 被引用的格在合并查询的结果里时,经过的每一次合并(见 MergeHop);别的查询为空。下钻的结论(status、version、
#: cell_source)说的是最后追到的那个输入的那一格,cell 仍是报告引用的那一格
merge: list[MergeHop] = field(default_factory=list)
# ==========================================================================
@@ -727,4 +755,11 @@ def contract_problems(out: ProvenanceOut) -> list[str]:
bad("table_only 时 version 必须有")
if out.status == "none" and out.version is not None:
bad("none 时 version 为 null")
if out.reason is not None and out.reason.code == "merge_no_lineage":
if out.status != "none":
bad("merge_no_lineage 时 status 为 none")
if not out.merge or out.merge[-1].input is not None:
bad("merge_no_lineage 时 merge 的最后一跳没有追到输入(input 为 null)")
if any(hop.input is None for hop in out.merge[:-1]):
bad("merge 里只有最后一跳可以没追到输入")
return out_problems
+18 -2
View File
@@ -65,6 +65,7 @@ from app.engine.evidence import (
table_fields,
)
from app.engine.expressions import CellError, column_kind, locate_cell
from app.engine.merge_query import MERGE_SOURCE
from app.engine.nodes.llm import _split_structured, _usage_of
from app.engine.state import message_text
from app.providers import catalog as pricing
@@ -314,10 +315,13 @@ class _Fetch:
def _hidden(snapshot: dict[str, Any], source: Any, masked: dict[str, Any] | None) -> set[str]:
"""要遮住的列(小写):查询当时记下的,加上数据源现在设的。"""
"""要遮住的列(小写):查询当时记下的,加上数据源现在设的。合并查询的结果不是数据源,按它合并的那几个数据源算。"""
names = _mask_names(snapshot.get("mask_columns"))
wanted = {str(source or "")}
if snapshot.get("source") == MERGE_SOURCE and isinstance(snapshot.get("sources"), list):
wanted |= {str(s) for s in snapshot["sources"]}
for name, cols in (masked or {}).items():
if str(name) == str(source or ""):
if str(name) in wanted:
names |= _mask_names(cols)
return names
@@ -349,6 +353,8 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter
head = f"【{alias}】查询结果({entry.get('tool') or '查询'} · 数据源 {source or '—'} · 共 {total} 行)"
sql = _clean(str(snapshot.get("sql") or ""))
lines = [head, f"SQL:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"]
if snapshot.get("source") == MERGE_SOURCE:
lines.append(_merge_line(snapshot))
hidden = _hidden(snapshot, source, masked)
if provenance_on and snapshot.get("data_version"):
lines += _provenance_lines(snapshot, fetch, hidden)
@@ -374,6 +380,16 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter
return "\n".join(lines)
def _merge_line(snapshot: dict[str, Any]) -> str:
"""合并查询结果的摘录多一行:合并了哪几个输入(SQL 里的别名、节点、数据源、行数)。上面的 SQL 就是合并 SQL。"""
parts = []
for item in snapshot.get("inputs") or []:
if isinstance(item, dict):
parts.append(f"{item.get('alias')}(节点 {item.get('node_id') or '—'},数据源 {item.get('source') or '—'},"
f"{item.get('rows')} 行)")
return "合并的输入:" + ("、".join(parts) if parts else "—")
def _provenance_lines(snapshot: dict[str, Any], fetch: _Fetch, hidden: set[str]) -> list[str]:
"""上传表格的查询快照的来历行(P4-SPEC 3.2、3.3):provenance.judge_lines 生成,这里只负责取冻结的表结构快照、
算 SQL 用到的表(direct_select.tables_in,和执行这条 SQL 的 SQLite 同一种读法,宁可多认),把现成的遮罩交过去。
+25
View File
@@ -395,6 +395,31 @@ async def test_a_computed_merge_column_has_no_cell_lineage(monkeypatch, sources,
"column": None}]
async def test_evidence_graph_links_the_merge_to_its_inputs(monkeypatch, sources, client):
row = await run_graph(monkeypatch)
doc = load_doc(row)
body = (await client.get(f"/api/runs/{row.id}/evidence")).json()
merged = sorted((e["to"], e["input"]) for e in body["edges"] if e["rel"] == "merged_from" and e["from"] == "Q3")
assert merged == sorted([(alias_of(doc, "q_sales"), "s"), (alias_of(doc, "q_visits"), "v")])
def test_judge_excerpt_names_the_inputs_and_masks_their_sources():
"""裁判看合并结果时,知道合并了哪几个输入;输入数据源现在设的遮罩照样生效。"""
from app.engine import judge
from app.engine.merge_query import MergeInput, execute
sales = {"columns": ["门店", "订单数"], "rows": [["S01", 3]], "truncated": False, "source": "merge_stores"}
visits = {"columns": ["门店", "手机号"], "rows": [["S01", "13800000000"]], "truncated": False,
"source": "merge_members"}
snap = execute([MergeInput("s", "q_sales", "a" * 64, sales), MergeInput("v", "q_visits", "b" * 64, visits)],
"SELECT s.门店, s.订单数, v.手机号 FROM s JOIN v ON s.门店 = v.门店").snapshot()
entry = {"kind": "query", "artifact": "m" * 64, "source": MERGE_SOURCE}
text = judge._query_excerpt("Q3", entry, {0}, ["订单数"], False, lambda _a: snap,
{"merge_members": ["手机号"]})
assert "合并的输入:s(节点 q_sales,数据源 merge_stores,1 行)、v(节点 q_visits,数据源 merge_members,1 行)" in text
assert "13800000000" not in text and "遮罩" in text
async def test_a_truncated_input_fails_the_node(monkeypatch, sources):
row = await run_graph(monkeypatch, store_graph(visits_args={"limit": 2}, report=False))
assert row.status == "failed"
+113
View File
@@ -0,0 +1,113 @@
"""合并查询的结果接进证据下钻(期 4「推断的来源」):宁可不下钻,也不下钻错。
上传表格(合成客流表,按配方导入,夹具同 test_provenance_api)查一次、手工源的气温快照现造一份,两者在库外按日期
合并。合并结果里直接来自客流表的格,下钻结论要和直接引用客流表那一格完全相同(同一个原表格子);合并 SQL 算出来的
列判为没有逐格来历;来自手工源的格照手工源的结论(not_upload)。每份结论都断言 contract_problems 为空。
"""
from __future__ import annotations
import pytest
from app.api import evidence as api
from app.core import artifact_store
from app.data.provenance_types import DOC_PROVENANCE, MergeHop
from app.engine.merge_query import MERGE_SOURCE, MergeInput, execute
from tests.fixtures.xlsx.flow import SHEET
from tests.test_provenance_api import ( # noqa: F401 夹具按名字注入(rep 依赖 upload_world、client)
cell_seg,
cell_value,
client,
ok,
put,
rep,
report_of,
run_sql,
upload_world,
)
FLOW_SQL = 'SELECT "日期", "全日客流" FROM "日客流" ORDER BY "日期" DESC LIMIT 6'
MERGE_SQL = ('SELECT f."日期", f."全日客流", w."气温", f."全日客流" * 2 AS 加倍 FROM f JOIN w ON f."日期" = w."日期" '
'ORDER BY f."日期"')
async def merged_world(up, c):
"""(文档, 封存范围, 合并结果, 客流快照)。客流按日期倒序查,合并结果按日期正序排:合并结果第 0 行是客流第 5 行。"""
qid, sid = await run_sql(c, up, FLOW_SQL)
flow = artifact_store.load(qid)
dates = sorted(r[0] for r in flow["rows"])
weather = {"columns": ["日期", "气温"], "rows": [[d, 20 + i] for i, d in enumerate(dates)], "row_count": len(dates),
"truncated": False, "sql": "SELECT 日期, 气温 FROM 每日气温", "source": "weather_demo",
"column_types": {"日期": "text", "气温": "number"}}
wid = await put(weather)
outcome = execute([MergeInput(alias="f", node_id="q_flow", artifact=qid, snapshot=flow),
MergeInput(alias="w", node_id="q_weather", artifact=wid, snapshot=weather)], MERGE_SQL)
mid = await put(outcome.snapshot())
doc = {"schema": "agentlab.report/1", "entity_syntax": 2, "run_id": "run-merge", "node_id": "write",
"provenance": DOC_PROVENANCE, "blocks": [], "catalog": {
"Q1": {"kind": "query", "artifact": qid, "tool": f"db_query__{up.name}", "node_id": "q_flow",
"source": up.name},
"Q2": {"kind": "query", "artifact": wid, "tool": "db_query__weather_demo", "node_id": "q_weather",
"source": "weather_demo"},
"Q3": {"kind": "query", "artifact": mid, "node_id": "merge", "source": MERGE_SOURCE}}}
seal = {"sealed": True, "ok": True, "manifest_seq": 9, "legacy": False}
sealed = api._Sealed(run_id="run-merge", graph={}, seal=seal, events=[], queries={
qid: {"node_id": "q_flow", "tool": f"db_query__{up.name}"},
wid: {"node_id": "q_weather", "tool": "db_query__weather_demo"},
mid: {"node_id": "merge", "source": MERGE_SOURCE}}, schemas={sid: {"node_id": "q_flow"}})
return doc, sealed, outcome, flow
async def drill(doc, sealed, *, alias, row, column):
return ok(await api.segment_provenance(report_of(doc), cell_seg(alias=alias, row=row, column=column), sealed,
masks=api._Masks()))
async def test_a_merged_upload_cell_points_at_the_same_original_cell(rep, client):
doc, sealed, outcome, flow = await merged_world(rep, client)
assert len(outcome.rows) == 6
for r in range(len(outcome.rows)):
via_merge = await drill(doc, sealed, alias="Q3", row=r, column="全日客流")
direct = await drill(doc, sealed, alias="Q1", row=5 - r, column="全日客流")
assert via_merge.status == direct.status == "inferred"
# 同一个原表格子、同一份数据版本:结论逐字相同,只是 cell 仍是报告引用的那一格,并记下经过的合并
assert via_merge.cell_source == direct.cell_source and via_merge.version == direct.version
assert (via_merge.cell.alias, via_merge.cell.row, via_merge.cell.column) == ("Q3", r, "全日客流")
assert via_merge.merge == [MergeHop(alias="Q3", node_id="merge", input="f", query="Q1", row=5 - r,
column="全日客流")]
# 原表那一格的值就是合并结果里这一格的值
assert cell_value(rep.raws[0], SHEET, via_merge.cell_source.cell) == outcome.rows[r][1]
async def test_a_computed_merge_column_is_not_drilled(rep, client):
doc, sealed, _outcome, _flow = await merged_world(rep, client)
out = await drill(doc, sealed, alias="Q3", row=0, column="加倍")
assert (out.status, out.reason.code, out.version, out.cell_source) == ("none", "merge_no_lineage", None, None)
assert out.merge == [MergeHop(alias="Q3", node_id="merge")]
assert "表级来历" in out.reason.text
async def test_a_manual_source_column_keeps_its_own_answer(rep, client):
doc, sealed, _outcome, _flow = await merged_world(rep, client)
out = await drill(doc, sealed, alias="Q3", row=0, column="气温")
assert (out.status, out.reason.code) == ("none", "not_upload")
assert out.merge == [MergeHop(alias="Q3", node_id="merge", input="w", query="Q2", row=0, column="气温")]
async def test_an_unsealed_merge_snapshot_is_not_drilled(rep, client):
doc, sealed, _outcome, _flow = await merged_world(rep, client)
sealed.queries.pop(doc["catalog"]["Q3"]["artifact"])
out = await drill(doc, sealed, alias="Q3", row=0, column="全日客流")
assert (out.status, out.reason.code) == ("none", "not_sealed")
@pytest.mark.parametrize("column, hinted", [("全日客流", True), ("气温", False), ("加倍", False)])
async def test_the_query_step_hint_follows_the_trace(rep, client, column, hinted):
"""界面只在追得到上传表格的格上请求推断来源;合并步骤后面接上追到的输入步骤。"""
doc, sealed, _outcome, _flow = await merged_world(rep, client)
steps = await api._query_steps(doc["catalog"]["Q3"]["artifact"], [(0, column)], doc, sealed, api._Masks(),
hint=True)
assert steps[0]["merge"]["inputs"][0]["query"] == "Q1"
assert ("provenance" in steps[0]) is hinted
assert len(steps) == (1 if column == "加倍" else 2)
if len(steps) == 2:
assert steps[1]["merged_into"] == "Q3" and "provenance" not in steps[1]
+13 -7
View File
@@ -44,7 +44,7 @@ def test_literals():
"legacy_doc", "not_cell", "not_sealed", "not_upload", "simple_upload", "manifest_unreadable",
"chain_mismatch", "expression", "alias", "multi_table", "unparsed", "no_pk", "pk_missing", "masked",
"null_value", "null_pk", "snapshot_gone", "db_tampered", "recheck_missing", "recheck_multiple",
"recheck_mismatch", "no_lineage"}
"recheck_mismatch", "no_lineage", "merge_no_lineage"}
assert len(get_args(P.ReasonCode)) == len(set(get_args(P.ReasonCode)))
assert set(get_args(P.AlertCode)) == {"db_tampered", "chain_mismatch", "manifest_unreadable"}
assert set(get_args(P.AlertCode)) <= set(get_args(P.ReasonCode))
@@ -130,7 +130,10 @@ SHAPES: dict[str, list[tuple[str, object]]] = {
("cell_status", None), ("acceptance", None), ("detail", None)],
"ProvenanceOut": [("schema", P.SCHEMA), ("report", REQ), ("segment", REQ), ("cell", None), ("status", REQ),
("reason", None), ("alert", None), ("sealed", REQ), ("version", None),
("cell_source", None), ("checks", FACTORY)],
("cell_source", None), ("checks", FACTORY), ("merge", FACTORY)],
# 合并查询结果里的格经过的每一次合并(P3:合并查询节点)
"MergeHop": [("alias", REQ), ("node_id", None), ("input", None), ("query", None), ("row", None),
("column", None)],
# 中间结构(P4-SPEC 7.2–7.4)
"DirectSelect": [("table", REQ), ("alias", REQ), ("columns", REQ)],
"SelectRefusal": [("code", REQ), ("detail", REQ)],
@@ -146,7 +149,7 @@ SHAPES: dict[str, list[tuple[str, object]]] = {
}
RESPONSE = ("ReportRef", "CellRef", "Reason", "Alert", "TableRef", "PeriodView", "AcceptanceView", "StateNote",
"PartView", "VersionView", "FromCell", "YearSource", "CanonicalView", "Recheck", "CellSource",
"RelatedCheck", "ProvenanceOut")
"RelatedCheck", "ProvenanceOut", "MergeHop")
@pytest.mark.parametrize("name", sorted(SHAPES))
@@ -184,7 +187,8 @@ def test_intermediate_types_take_positional_args_as_the_spec_writes_them():
KEYS = {
"top": ["schema", "report", "segment", "cell", "status", "reason", "alert", "sealed", "version", "cell_source",
"checks"],
"checks", "merge"],
"merge": ["alias", "node_id", "input", "query", "row", "column"],
"report": ["node_id", "doc_artifact"],
"cell": ["alias", "row", "column", "artifact"],
"reason": ["code", "detail", "text"],
@@ -248,6 +252,7 @@ def full_out() -> P.ProvenanceOut:
checks=[P.RelatedCheck(id="R1", kind="relation_sum_eq", title="「日客流」:全日客流 = 分区甲 + 分区乙",
part_status="mismatch", row_status="passed", cell_status=None,
acceptance=_accept(), detail="合成的原因摘要")],
merge=[P.MergeHop(alias="Q3", node_id="merge", input="f", query="Q1", row=5, column="全日客流")],
)
@@ -258,7 +263,7 @@ def test_asdict_matches_the_spec_json_level_by_level():
"top": d, "report": d["report"], "cell": d["cell"], "version": v, "table": v["tables"][0], "part": part,
"period": part["period"], "acceptance": part["acceptances"][0], "state": part["purged"],
"cell_source": cs, "from": cs["from"][0], "year": cs["year"], "canonical": cs["canonical"],
"recheck": cs["recheck"], "check": d["checks"][0],
"recheck": cs["recheck"], "check": d["checks"][0], "merge": d["merge"][0],
}
for name, obj in levels.items():
assert isinstance(obj, dict), name
@@ -280,7 +285,8 @@ def test_null_and_empty_defaults_match_the_spec():
reason=P.refuse("legacy_doc"), sealed=False)
d = asdict(out)
assert d["schema"] == "agentlab.provenance/1"
assert (d["cell"], d["alert"], d["version"], d["cell_source"], d["checks"]) == (None, None, None, None, [])
assert (d["cell"], d["alert"], d["version"], d["cell_source"], d["checks"], d["merge"]) == (
None, None, None, None, [], [])
assert d["reason"] == {"code": "legacy_doc", "detail": "", "text": P.REASON_TEXT["legacy_doc"]}
@@ -666,7 +672,7 @@ TS_INTERFACES = {
"ProvenancePeriod": "PeriodView", "ProvenanceAcceptance": "AcceptanceView", "ProvenanceStateNote": "StateNote",
"ProvenancePart": "PartView", "ProvenanceVersion": "VersionView", "ProvenanceFromCell": "FromCell",
"ProvenanceYear": "YearSource", "ProvenanceCanonical": "CanonicalView", "ProvenanceRecheck": "Recheck",
"ProvenanceCellSource": "CellSource", "ProvenanceCheck": "RelatedCheck",
"ProvenanceCellSource": "CellSource", "ProvenanceCheck": "RelatedCheck", "ProvenanceMergeHop": "MergeHop",
}
TS_UNIONS = {
"ProvenanceStatus": P.Status, "ProvenanceReasonCode": P.ReasonCode, "ProvenanceAlertCode": P.AlertCode,
@@ -6639,7 +6639,8 @@
"acceptance": null,
"detail": null
}
]
],
"merge": []
},
"s8": {
"schema": "agentlab.provenance/1",
@@ -6793,7 +6794,8 @@
"acceptance": null,
"detail": null
}
]
],
"merge": []
},
"s12": {
"schema": "agentlab.provenance/1",
@@ -6952,7 +6954,8 @@
"acceptance": null,
"detail": null
}
]
],
"merge": []
},
"s16": {
"schema": "agentlab.provenance/1",
@@ -7018,7 +7021,8 @@
]
},
"cell_source": null,
"checks": []
"checks": [],
"merge": []
},
"s20": {
"schema": "agentlab.provenance/1",
@@ -7084,7 +7088,8 @@
]
},
"cell_source": null,
"checks": []
"checks": [],
"merge": []
},
"s24": {
"schema": "agentlab.provenance/1",
@@ -7150,7 +7155,8 @@
]
},
"cell_source": null,
"checks": []
"checks": [],
"merge": []
},
"s28": {
"schema": "agentlab.provenance/1",
@@ -7216,7 +7222,8 @@
]
},
"cell_source": null,
"checks": []
"checks": [],
"merge": []
},
"s56": {
"schema": "agentlab.provenance/1",
@@ -7282,7 +7289,8 @@
]
},
"cell_source": null,
"checks": []
"checks": [],
"merge": []
},
"s60": {
"schema": "agentlab.provenance/1",
@@ -7466,7 +7474,8 @@
"acceptance": null,
"detail": null
}
]
],
"merge": []
},
"s64": {
"schema": "agentlab.provenance/1",
@@ -7642,7 +7651,8 @@
"acceptance": null,
"detail": null
}
]
],
"merge": []
},
"s68": {
"schema": "agentlab.provenance/1",
@@ -7877,7 +7887,8 @@
"acceptance": null,
"detail": null
}
]
],
"merge": []
},
"s72": {
"schema": "agentlab.provenance/1",
@@ -8047,7 +8058,8 @@
"acceptance": null,
"detail": null
}
]
],
"merge": []
},
"s76": {
"schema": "agentlab.provenance/1",
@@ -8217,7 +8229,8 @@
"acceptance": null,
"detail": null
}
]
],
"merge": []
},
"s84": {
"schema": "agentlab.provenance/1",
@@ -8357,7 +8370,8 @@
"acceptance": null,
"detail": null
}
]
],
"merge": []
},
"s88": {
"schema": "agentlab.provenance/1",
@@ -8447,7 +8461,8 @@
"ok": true
}
},
"checks": []
"checks": [],
"merge": []
},
"s92": {
"schema": "agentlab.provenance/1",
@@ -8472,7 +8487,8 @@
"sealed": true,
"version": null,
"cell_source": null,
"checks": []
"checks": [],
"merge": []
},
"s52": {
"schema": "agentlab.provenance/1",
@@ -8626,7 +8642,8 @@
"acceptance": null,
"detail": null
}
]
],
"merge": []
},
"s32": {
"schema": "agentlab.provenance/1",
@@ -8695,7 +8712,8 @@
]
},
"cell_source": null,
"checks": []
"checks": [],
"merge": []
},
"s36": {
"schema": "agentlab.provenance/1",
@@ -8764,7 +8782,8 @@
]
},
"cell_source": null,
"checks": []
"checks": [],
"merge": []
},
"s48": {
"schema": "agentlab.provenance/1",
@@ -8830,7 +8849,8 @@
]
},
"cell_source": null,
"checks": []
"checks": [],
"merge": []
},
"s44": {
"schema": "agentlab.provenance/1",
@@ -8858,7 +8878,8 @@
"sealed": true,
"version": null,
"cell_source": null,
"checks": []
"checks": [],
"merge": []
},
"s40": {
"schema": "agentlab.provenance/1",
@@ -9016,7 +9037,8 @@
"acceptance": null,
"detail": null
}
]
],
"merge": []
},
"s80": {
"schema": "agentlab.provenance/1",
@@ -9190,7 +9212,8 @@
"acceptance": null,
"detail": null
}
]
],
"merge": []
}
},
"errors": {
+16 -1
View File
@@ -2262,7 +2262,7 @@ export type ProvenanceReasonCode =
| 'legacy_doc' | 'not_cell' | 'not_sealed' | 'not_upload' | 'simple_upload' | 'manifest_unreadable'
| 'chain_mismatch' | 'expression' | 'alias' | 'multi_table' | 'unparsed' | 'no_pk' | 'pk_missing' | 'masked'
| 'null_value' | 'null_pk' | 'snapshot_gone' | 'db_tampered' | 'recheck_missing' | 'recheck_multiple'
| 'recheck_mismatch' | 'no_lineage'
| 'recheck_mismatch' | 'no_lineage' | 'merge_no_lineage'
/** 标红的提示:出现时 reason.code 与它相同,界面只画提示 */
export type ProvenanceAlertCode = 'db_tampered' | 'chain_mismatch' | 'manifest_unreadable'
@@ -2470,4 +2470,19 @@ export interface EvidenceProvenance {
version: ProvenanceVersion | null
cell_source: ProvenanceCellSource | null
checks: ProvenanceCheck[]
/** 被引用的格在合并查询的结果里时,经过的每一次合并;别的查询为空 */
merge: ProvenanceMergeHop[]
}
/** 经过的一次合并查询:合并结果里被引用的格追到了哪个输入的哪一格;追不到的那一跳后四项为 null */
export interface ProvenanceMergeHop {
/** 合并结果在目录里的编号(Q3) */
alias: string
node_id: string | null
/** 追到的输入别名(合并 SQL 里的表名) */
input: string | null
/** 那个输入在目录里的编号(Q1) */
query: string | null
row: number | null
column: string | null
}