From 8f75849a1c78d3c4259bf5d66e6fc62b251614dd Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 3 Oct 2026 09:36:20 +0800 Subject: [PATCH] =?UTF-8?q?=E5=90=88=E5=B9=B6=E6=9F=A5=E8=AF=A2=E8=8A=82?= =?UTF-8?q?=E7=82=B9=EF=BC=88=E4=BA=8C=EF=BC=89=EF=BC=9A=E8=AF=81=E6=8D=AE?= =?UTF-8?q?=E9=9D=A2=E6=9D=BF=E5=B1=95=E7=A4=BA=E5=90=88=E5=B9=B6=E7=9A=84?= =?UTF-8?q?=E8=BE=93=E5=85=A5=E3=80=81=E5=90=88=E5=B9=B6=20SQL=20=E5=92=8C?= =?UTF-8?q?=E8=AD=A6=E5=91=8A=EF=BC=8C=E8=A2=AB=E5=BC=95=E7=94=A8=E7=9A=84?= =?UTF-8?q?=E6=A0=BC=E6=B2=BF=E9=80=90=E6=A0=BC=E6=9D=A5=E5=8E=86=E8=BF=BD?= =?UTF-8?q?=E5=88=B0=E8=BE=93=E5=85=A5=E7=9A=84=E9=82=A3=E4=B8=80=E6=A0=BC?= =?UTF-8?q?=E5=B9=B6=E6=8E=A5=E4=B8=8A=E8=BE=93=E5=85=A5=E7=9A=84=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=AD=A5=E9=AA=A4=EF=BC=9B=E6=8E=A8=E6=96=AD=E6=9D=A5?= =?UTF-8?q?=E6=BA=90=E7=A9=BF=E8=BF=87=E5=90=88=E5=B9=B6=E4=B8=8B=E9=92=BB?= =?UTF-8?q?=E5=88=B0=E5=8E=9F=E8=A1=A8=E6=A0=BC=E5=AD=90=EF=BC=8C=E8=BF=BD?= =?UTF-8?q?=E4=B8=8D=E5=88=B0=E7=9A=84=E5=88=A4=E4=B8=BA=E6=B2=A1=E6=9C=89?= =?UTF-8?q?=E9=80=90=E6=A0=BC=E6=9D=A5=E5=8E=86=EF=BC=9B=E8=A3=81=E5=88=A4?= =?UTF-8?q?=E6=91=98=E5=BD=95=E5=86=99=E6=98=8E=E5=90=88=E5=B9=B6=E7=9A=84?= =?UTF-8?q?=E8=BE=93=E5=85=A5=E5=B9=B6=E6=8C=89=E5=BA=95=E5=B1=82=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=BA=90=E9=81=AE=E7=BD=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/evidence.py | 205 +++++++++++++++++- backend/app/data/provenance_types.py | 35 +++ backend/app/engine/judge.py | 20 +- backend/tests/test_merge_node.py | 25 +++ backend/tests/test_merge_provenance.py | 113 ++++++++++ backend/tests/test_provenance_contract.py | 20 +- .../run/__tests__/evidence-provenance.json | 69 ++++-- frontend/src/types.ts | 17 +- 8 files changed, 460 insertions(+), 44 deletions(-) create mode 100644 backend/tests/test_merge_provenance.py diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index 3e2fefec..a69ea0b5 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -58,7 +58,9 @@ from app.data.provenance_types import ( CellRef, Chain, ChainProblem, + REASON_TEXT, LocateProblem, + MergeHop, ProvenanceOut, RawState, Reason, @@ -100,6 +102,7 @@ from app.engine.evidence import ( uncited_claims, ) from app.engine.judge import FIELDS_MAX, SETTLED, UNSUPPORTED, VERDICTS, candidates, outdated +from app.engine.merge_query import MERGE_SOURCE, trace_cell from app.engine.toolcalls import QUERY_PREFIX router = APIRouter(prefix="/api/runs", tags=["evidence"]) @@ -426,6 +429,12 @@ def _graph_of(report: _Report, sealed: _Sealed, inputs: dict[str, Any]) -> tuple if entry.get("kind") in ("query", "retrieval"): # 查询、检索条目带上是什么、有多大:图上不点开也看得出这是哪次取数 item.update({k: entry[k] for k in ("tool", "source", "columns", "rows", "truncated") if k in entry}) + if entry.get("kind") == "query" and entry.get("source") == MERGE_SOURCE: + # 合并结果连向它合并的那几次查询:图上看得出 Q3 是 Q1、Q2 按键合并出来的 + edges.extend({"from": alias, "to": target, "rel": "merged_from", "input": source.get("alias"), + "report": report.node_id} + for source in _merge_inputs(entry.get("artifact")) + if (target := _query_alias(catalog, source.get("artifact")))) elif entry.get("kind") in ENTITY_KINDS: # 表和字段:叫什么、出现在哪几次查询里、来历(表结构快照 / SQL / 结果列)各自封存了没有 item.update({k: entry[k] for k in ("name", "table", "tables", "qualified", "source") if entry.get(k)}) @@ -454,6 +463,16 @@ def _graph_of(report: _Report, sealed: _Sealed, inputs: dict[str, Any]) -> tuple return evidence, edges +def _merge_inputs(artifact: Any) -> list[dict[str, Any]]: + """合并结果快照记下的输入;取不回、哈希不符的当没有(证据图只是少几条边,片段接口照样说清楚)。""" + try: + snap = load(str(artifact)) if artifact else None + except ValueError: + return [] + inputs = snap.get("inputs") if isinstance(snap, dict) else None + return [i for i in inputs if isinstance(i, dict)] if isinstance(inputs, list) else [] + + def _cell_name(alias: Any, locator: dict[str, Any]) -> str: """Q1.r0.gmv:报告目录里的全局编号加行列,和 [[v:]] 的写法一致。""" if not alias or not isinstance(locator.get("row"), int) or not locator.get("column"): @@ -778,8 +797,8 @@ async def _chain(seg: dict[str, Any], doc: dict[str, Any], sealed: _Sealed, locator = cite.get("locator") or {} # 只有直接引用的单元格才可能推断来源(P4-SPEC 2.8.2)。指标链里的查询步骤不带:指标片段的推断来源只会 # 回「不是单元格」,带上提示等于让界面白发一次请求 - return [await _query_step(str(entry.get("artifact") or ""), [(locator.get("row"), locator.get("column"))], - doc, sealed, masks, hint=doc.get("provenance") == DOC_PROVENANCE)] + return await _query_steps(str(entry.get("artifact") or ""), [(locator.get("row"), locator.get("column"))], + doc, sealed, masks, hint=doc.get("provenance") == DOC_PROVENANCE) if cite.get("kind") == "input": name = (cite.get("locator") or {}).get("field") inputs = _input_payloads(sealed) @@ -843,7 +862,7 @@ async def _metric_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[s one["cell"] = cell wanted.setdefault(str(artifact_of), []).extend(_located(item.get("locator") or {})) inputs.append(one) - queries = [await _query_step(a, cells, doc, sealed, masks) for a, cells in wanted.items()] + queries = [s for a, cells in wanted.items() for s in await _query_steps(a, cells, doc, sealed, masks)] return [step, *inputs, *queries] @@ -932,20 +951,30 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str columns = [str(c) for c in snap.get("columns") or []] rows = snap["rows"] step["source"] = step["source"] or snap.get("source") - # 遮的是「查询当时记下的」和「数据源现在设的」两者之和:事后加的遮罩照样生效,数据源改名、 - # 删掉了也不会把当时遮着的列亮出来 - hidden, found = await masks.of(step["source"]) - recorded = snap.get("mask_columns") - hidden = hidden | ({str(c).lower() for c in recorded if isinstance(c, str)} if isinstance(recorded, list) else set()) - if not found: - step["mask_note"] = SOURCE_GONE.format(source=step["source"]) + merged = snap.get("source") == MERGE_SOURCE + if merged: + # 合并查询不是数据源:遮罩按它合并的那几个数据源算(_merge_masks),不能拿「合并查询」去找数据源, + # 找不到再说它「已不存在」 + hidden, gone = await _merge_masks(snap, masks) + if gone: + step["mask_note"] = SOURCE_GONE.format(source="、".join(gone)) + step["merge"] = _merge_view(snap, cells, catalog, sealed) + if hint and _leads_to_upload(snap, cells): + step["provenance"] = True + else: + # 遮的是「查询当时记下的」和「数据源现在设的」两者之和:事后加的遮罩照样生效,数据源改名、 + # 删掉了也不会把当时遮着的列亮出来 + hidden, found = await masks.of(step["source"]) + hidden = hidden | _recorded_masks(snap) + if not found: + step["mask_note"] = SOURCE_GONE.format(source=step["source"]) masked = {i for i, c in enumerate(columns) if c.lower() in hidden} index = sorted({i for r in rows_hit for i in range(r - WINDOW, r + WINDOW + 1) if 0 <= i < len(rows)}) step.update(sql=snap.get("sql"), columns=columns, total_rows=len(rows), truncated=bool(snap.get("truncated")), masked=[columns[i] for i in sorted(masked)], # 查询时从驱动的原始值记下的列类型(老快照没有):文本列里的 "2026" 按文本显示,不当数 column_types=snap.get("column_types") if isinstance(snap.get("column_types"), dict) else {}, - **({"provenance": True} if hint and snap.get("data_version") else {})) + **({"provenance": True} if hint and snap.get("data_version") and not merged else {})) if len(index) > MAX_WINDOW_ROWS: index, step["window_truncated"] = index[:MAX_WINDOW_ROWS], True step["row_index"] = index @@ -955,6 +984,116 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str return step +# -------------------------------------------------------------------------- +# 合并查询的结果:表级来历(输入、合并 SQL)和逐格来历(被引用的格追到输入的哪一格) +# -------------------------------------------------------------------------- + +#: 一条链最多追几层合并(合并的合并)。画布上不会无限嵌套,这只是防御坏快照 +MERGE_DEPTH = 4 + + +async def _query_steps(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str, Any], sealed: _Sealed, + masks: _Masks, *, hint: bool = False, depth: int = 0) -> list[dict[str, Any]]: + """查询步骤;快照是合并结果时,后面接上被引用的格追到的那几个输入的查询步骤(高亮追到的那一格)。 + + 追不到的格不接任何输入步骤:合并步骤的 merge.traced 里照实写「没有逐格来历」,只给表级来历。输入步骤带 + merged_into(合并结果的编号),界面据此标明它是哪次合并的输入。不是合并结果的快照,返回的就是原来那一个步骤。 + """ + step = await _query_step(artifact, cells, doc, sealed, masks, hint=hint) + traced = [t for t in (step.get("merge") or {}).get("traced") or [] if t.get("input")] + if not traced or depth >= MERGE_DEPTH: + return [step] + by_input: dict[str, list[tuple[Any, Any]]] = {} + artifacts = {str(i.get("alias")): str(i.get("artifact") or "") for i in step["merge"]["inputs"]} + for t in traced: + by_input.setdefault(artifacts.get(str(t["input"]), ""), []).append((t["row"], t["column"])) + out = [step] + for source, picked in by_input.items(): + if not source: + continue + for sub in await _query_steps(source, picked, doc, sealed, masks, depth=depth + 1): + out.append({**sub, "merged_into": sub.get("merged_into") or step.get("alias")}) + return out + + +def _merge_view(snap: dict[str, Any], cells: list[tuple[Any, Any]], catalog: dict[str, Any], + sealed: _Sealed) -> dict[str, Any]: + """合并步骤的 merge 一节:每个输入(别名、节点、节点名、目录编号、行数、数据源、封存与否)、合并 SQL、执行时的 + 警告,以及被引用的每一格追到了哪个输入的哪一格(追不到的写明原因)。""" + labels = {str(n.get("id")): (n.get("data") or {}).get("label") or None + for n in sealed.graph.get("nodes") or [] if isinstance(n, dict)} + inputs = [] + for item in snap.get("inputs") or []: + if not isinstance(item, dict): + continue + art = str(item.get("artifact") or "") + inputs.append({"alias": item.get("alias"), "node_id": item.get("node_id"), + "label": labels.get(str(item.get("node_id"))), "query": _query_alias(catalog, art), + "rows": item.get("rows"), "source": item.get("source"), "artifact": art or None, + "sealed": sealed.trusted and art in sealed.queries}) + traced = [] + for row, column in dict.fromkeys((r, c) for r, c in cells if isinstance(r, int) and c): + hit = trace_cell(snap, row, column) + if hit is None: + traced.append({"cell": [row, str(column)], "input": None, "query": None, "row": None, "column": None, + "note": REASON_TEXT["merge_no_lineage"]}) + else: + traced.append({"cell": [row, str(column)], "input": hit.alias, "query": _query_alias(catalog, hit.artifact), + "row": hit.row, "column": hit.column}) + warnings = [{"code": w.get("code"), "message": w.get("message")} for w in snap.get("warnings") or [] + if isinstance(w, dict) and w.get("message")] + return {"sql": snap.get("sql"), "inputs": inputs, "warnings": warnings, "traced": traced} + + +async def _merge_masks(snap: dict[str, Any], masks: _Masks) -> tuple[set[str], list[str]]: + """合并结果要遮的列(小写)和已经不在的数据源: + + - 合并当时记下的(快照的 mask_columns:各输入当时的遮罩列,加上来自遮罩列、换了名字的结果列); + - 它合并的每个数据源现在设的(按列名); + - 结果列有逐格来历、来自某个数据源现在遮着的列的,按来历遮(换了名字也遮住)。宁可多遮。""" + hidden = _recorded_masks(snap) + gone: list[str] = [] + for name in snap.get("sources") or []: + cols, found = await masks.of(name) + hidden |= cols + if not found: + gone.append(str(name)) + lineage = snap.get("lineage") if isinstance(snap.get("lineage"), dict) else {} + origins = lineage.get("sources") or [] + by_alias = {str(i.get("alias")): i for i in snap.get("inputs") or [] if isinstance(i, dict)} + for name, target in zip(snap.get("columns") or [], lineage.get("columns") or []): + if not isinstance(target, dict) or not isinstance(target.get("source"), int) \ + or not 0 <= target["source"] < len(origins): + continue + origin = origins[target["source"]] if isinstance(origins[target["source"]], dict) else {} + source = (by_alias.get(str(origin.get("input"))) or {}).get("source") + if source and source != MERGE_SOURCE and str(target.get("column") or "").lower() in (await masks.of(source))[0]: + hidden.add(str(name).lower()) + return hidden, gone + + +def _leads_to_upload(snap: dict[str, Any], cells: list[tuple[Any, Any]], depth: int = 0) -> bool: + """被引用的格经合并追下去,最后落在上传表格的查询快照上(有 data_version):界面才去请求推断来源。""" + if depth >= MERGE_DEPTH: + return False + for row, column in cells: + hit = trace_cell(snap, row, column) + if hit is None or not hit.artifact: + continue + try: + source = load(hit.artifact) + except ValueError: + continue + if not isinstance(source, dict): + continue + if source.get("source") == MERGE_SOURCE: + if _leads_to_upload(source, [(hit.row, hit.column)], depth + 1): + return True + elif source.get("data_version"): + return True + return False + + # -------------------------------------------------------------------------- # GET /api/runs/{run_id}/evidence/segments/{segment_id}/provenance(期 4:推断的来源) # -------------------------------------------------------------------------- @@ -1070,6 +1209,10 @@ async def segment_provenance(report: _Report, seg: dict[str, Any], sealed: _Seal # R0:期 4 之前组装的文档(没有标记)一律不推断,期 4 之前的面板不变(1.4) if doc.get("provenance") != DOC_PROVENANCE: return none("legacy_doc") + # 合并查询结果里的格:先按逐格来历追到输入的那一格,再对那一格照常判断;追不到就只有表级来历 + if (cite.get("kind") == "cell" and cite.get("status") == "resolved" and seg.get("state") == "deterministic" + and entry.get("kind") == "query" and entry.get("source") == MERGE_SOURCE): + return await _merge_provenance(report, seg, entry, sealed, masks=masks, base=base) # R1:只对直接引用的查询单元格下钻;代码节点、Agent 字段的取数(工具名不是 db_query__ 开头)不算 if (cite.get("kind") != "cell" or cite.get("status") != "resolved" or seg.get("state") != "deterministic" or entry.get("kind") != "query" or not str(entry.get("tool") or "").startswith(QUERY_PREFIX)): @@ -1094,6 +1237,46 @@ async def segment_provenance(report: _Report, seg: dict[str, Any], sealed: _Seal masks=masks, sealed=sealed) +async def _merge_provenance(report: _Report, seg: dict[str, Any], entry: dict[str, Any], sealed: _Sealed, *, + masks: _Masks, base: dict[str, Any], hops: tuple[MergeHop, ...] = ()) -> ProvenanceOut: + """合并结果里的一格:沿合并快照记下的逐格来历(engine/merge_query.py)追到输入的那一格,换成对那一格的判断。 + + 宁可不下钻,也不下钻错:追不到(计算、聚合出来的列,合并 SQL 超出可追溯的写法,一模一样的两行)就判 + merge_no_lineage,只有表级来历(合并的输入、合并 SQL 在查询步骤的 merge 一节里)。追到的输入是数据源查询时, + 结论就是直接引用那一格时的结论(上传表格照常推断原表格子,手工源 not_upload);输入本身也是合并结果时再追一跳。 + cell 始终是报告引用的那一格,merge 列出经过的每一跳。 + """ + def out(status: str, reason: Reason, *more: MergeHop) -> ProvenanceOut: + return ProvenanceOut(status=status, reason=reason, merge=[*hops, *more], **base) # type: ignore[arg-type] + + cite = seg.get("cite") or {} + artifact = str(entry.get("artifact") or "") + snap, ok = await _Artifacts().get(artifact) if artifact in sealed.queries else (None, None) + if ok is not True or not isinstance(snap, dict): + return out("none", refuse("not_sealed")) + locator = cite.get("locator") or {} + row, column = locator.get("row"), locator.get("column") + if isinstance(column, str) and column.lower() in _recorded_masks(snap): + return out("none", refuse("masked")) + hit = trace_cell(snap, row, column) + catalog = (report.doc or {}).get("catalog") or {} + alias = _query_alias(catalog, hit.artifact) if hit is not None else None + hop = MergeHop(alias=str(cite.get("alias") or ""), node_id=entry.get("node_id"), + input=hit.alias if hit else None, query=alias, row=hit.row if hit else None, + column=hit.column if hit else None) + if hit is None or len(hops) >= MERGE_DEPTH: + return out("none", refuse("merge_no_lineage"), replace(hop, input=None, query=None, row=None, column=None)) + if alias is None: + # 输入不在这份报告的目录里(不在报告的上游、没有记进台账):认不出它是封存过的查询 + return out("none", refuse("not_sealed"), hop) + sub_entry = catalog[alias] + sub_seg = {**seg, "cite": {**cite, "alias": alias, "locator": {"row": hit.row, "column": hit.column}}} + if sub_entry.get("source") == MERGE_SOURCE: + return await _merge_provenance(report, sub_seg, sub_entry, sealed, masks=masks, base=base, hops=(*hops, hop)) + found = await segment_provenance(report, sub_seg, sealed, masks=masks) + return replace(found, cell=base["cell"], segment=base["segment"], merge=[*hops, hop]) + + async def _drill(base: dict[str, Any], *, seg: dict[str, Any], entry: dict[str, Any], query: dict[str, Any], schema: dict[str, Any], artifacts: _Artifacts, masks: _Masks, sealed: _Sealed) -> ProvenanceOut: """R5–R17(P4-SPEC 2.3):第一条不满足的就是结论。 diff --git a/backend/app/data/provenance_types.py b/backend/app/data/provenance_types.py index 703d49d6..b76802df 100644 --- a/backend/app/data/provenance_types.py +++ b/backend/app/data/provenance_types.py @@ -95,6 +95,8 @@ ReasonCode = Literal[ "recheck_missing", "recheck_multiple", "recheck_mismatch", # R17 "no_lineage", + # 合并查询的结果:这一格说不清来自哪个输入的哪一格(计算、聚合出来的列,或合并 SQL 超出可追溯的写法) + "merge_no_lineage", ] #: 标红的提示。出现时 reason.code 与它相同,界面只画提示、不再画 reason.text(P4-SPEC 4.1、4.2 末尾) @@ -179,6 +181,8 @@ REASON_TEXT: dict[str, str] = { "recheck_multiple": _RECHECK_TEXT, "recheck_mismatch": _RECHECK_TEXT, "no_lineage": "导入清单里没有这一格的溯源记录,只给出表级来历", + "merge_no_lineage": ("这一格来自合并查询,无法确定它对应哪个输入的哪一格(计算、聚合出来的列,或合并 SQL 超出可追溯的" + "写法),没有逐格来历,只有表级来历:合并了哪几个输入、用的哪条合并 SQL"), } #: 标红提示的原文:与同名 reason 的原文相同(两处同时出现时界面只画提示) @@ -506,6 +510,27 @@ class RelatedCheck: detail: str | None = None +@dataclass(kw_only=True) +class MergeHop: + """经过的一次合并查询(engine/merge_query.py):被引用的格在合并结果里,追到了哪个输入的哪一格。 + + 追不到的那一跳 input、query、row、column 都是 None,结论是 merge_no_lineage。输入本身也是合并结果时再追一跳, + ProvenanceOut.merge 按从报告引用的那一份往下的顺序列出每一跳。""" + + #: 合并结果在目录里的编号(Q3) + alias: str + #: 合并查询节点 + node_id: str | None = None + #: 追到的输入别名(合并 SQL 里的表名) + input: str | None = None + #: 那个输入在目录里的编号(Q1);输入不在这份报告的目录里时为 None + query: str | None = None + #: 输入快照里的行号,从 0 数 + row: int | None = None + #: 输入快照里的列名 + column: str | None = None + + @dataclass(kw_only=True) class ProvenanceOut: """推断来源接口的响应。asdict 之后就是 P4-SPEC 2.8.1 的 JSON;不变式见 contract_problems。""" @@ -523,6 +548,9 @@ class ProvenanceOut: version: VersionView | None = None cell_source: CellSource | None = None checks: list[RelatedCheck] = field(default_factory=list) + #: 被引用的格在合并查询的结果里时,经过的每一次合并(见 MergeHop);别的查询为空。下钻的结论(status、version、 + #: cell_source)说的是最后追到的那个输入的那一格,cell 仍是报告引用的那一格 + merge: list[MergeHop] = field(default_factory=list) # ========================================================================== @@ -727,4 +755,11 @@ def contract_problems(out: ProvenanceOut) -> list[str]: bad("table_only 时 version 必须有") if out.status == "none" and out.version is not None: bad("none 时 version 为 null") + if out.reason is not None and out.reason.code == "merge_no_lineage": + if out.status != "none": + bad("merge_no_lineage 时 status 为 none") + if not out.merge or out.merge[-1].input is not None: + bad("merge_no_lineage 时 merge 的最后一跳没有追到输入(input 为 null)") + if any(hop.input is None for hop in out.merge[:-1]): + bad("merge 里只有最后一跳可以没追到输入") return out_problems diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index b505d883..6ad9c5a4 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -65,6 +65,7 @@ from app.engine.evidence import ( table_fields, ) from app.engine.expressions import CellError, column_kind, locate_cell +from app.engine.merge_query import MERGE_SOURCE from app.engine.nodes.llm import _split_structured, _usage_of from app.engine.state import message_text from app.providers import catalog as pricing @@ -314,10 +315,13 @@ class _Fetch: def _hidden(snapshot: dict[str, Any], source: Any, masked: dict[str, Any] | None) -> set[str]: - """要遮住的列(小写):查询当时记下的,加上数据源现在设的。""" + """要遮住的列(小写):查询当时记下的,加上数据源现在设的。合并查询的结果不是数据源,按它合并的那几个数据源算。""" names = _mask_names(snapshot.get("mask_columns")) + wanted = {str(source or "")} + if snapshot.get("source") == MERGE_SOURCE and isinstance(snapshot.get("sources"), list): + wanted |= {str(s) for s in snapshot["sources"]} for name, cols in (masked or {}).items(): - if str(name) == str(source or ""): + if str(name) in wanted: names |= _mask_names(cols) return names @@ -349,6 +353,8 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter head = f"【{alias}】查询结果({entry.get('tool') or '查询'} · 数据源 {source or '—'} · 共 {total} 行)" sql = _clean(str(snapshot.get("sql") or "")) lines = [head, f"SQL:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"] + if snapshot.get("source") == MERGE_SOURCE: + lines.append(_merge_line(snapshot)) hidden = _hidden(snapshot, source, masked) if provenance_on and snapshot.get("data_version"): lines += _provenance_lines(snapshot, fetch, hidden) @@ -374,6 +380,16 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter return "\n".join(lines) +def _merge_line(snapshot: dict[str, Any]) -> str: + """合并查询结果的摘录多一行:合并了哪几个输入(SQL 里的别名、节点、数据源、行数)。上面的 SQL 就是合并 SQL。""" + parts = [] + for item in snapshot.get("inputs") or []: + if isinstance(item, dict): + parts.append(f"{item.get('alias')}(节点 {item.get('node_id') or '—'},数据源 {item.get('source') or '—'}," + f"{item.get('rows')} 行)") + return "合并的输入:" + ("、".join(parts) if parts else "—") + + def _provenance_lines(snapshot: dict[str, Any], fetch: _Fetch, hidden: set[str]) -> list[str]: """上传表格的查询快照的来历行(P4-SPEC 3.2、3.3):provenance.judge_lines 生成,这里只负责取冻结的表结构快照、 算 SQL 用到的表(direct_select.tables_in,和执行这条 SQL 的 SQLite 同一种读法,宁可多认),把现成的遮罩交过去。 diff --git a/backend/tests/test_merge_node.py b/backend/tests/test_merge_node.py index 9c1ee4d8..28019acb 100644 --- a/backend/tests/test_merge_node.py +++ b/backend/tests/test_merge_node.py @@ -395,6 +395,31 @@ async def test_a_computed_merge_column_has_no_cell_lineage(monkeypatch, sources, "column": None}] +async def test_evidence_graph_links_the_merge_to_its_inputs(monkeypatch, sources, client): + row = await run_graph(monkeypatch) + doc = load_doc(row) + body = (await client.get(f"/api/runs/{row.id}/evidence")).json() + merged = sorted((e["to"], e["input"]) for e in body["edges"] if e["rel"] == "merged_from" and e["from"] == "Q3") + assert merged == sorted([(alias_of(doc, "q_sales"), "s"), (alias_of(doc, "q_visits"), "v")]) + + +def test_judge_excerpt_names_the_inputs_and_masks_their_sources(): + """裁判看合并结果时,知道合并了哪几个输入;输入数据源现在设的遮罩照样生效。""" + from app.engine import judge + from app.engine.merge_query import MergeInput, execute + + sales = {"columns": ["门店", "订单数"], "rows": [["S01", 3]], "truncated": False, "source": "merge_stores"} + visits = {"columns": ["门店", "手机号"], "rows": [["S01", "13800000000"]], "truncated": False, + "source": "merge_members"} + snap = execute([MergeInput("s", "q_sales", "a" * 64, sales), MergeInput("v", "q_visits", "b" * 64, visits)], + "SELECT s.门店, s.订单数, v.手机号 FROM s JOIN v ON s.门店 = v.门店").snapshot() + entry = {"kind": "query", "artifact": "m" * 64, "source": MERGE_SOURCE} + text = judge._query_excerpt("Q3", entry, {0}, ["订单数"], False, lambda _a: snap, + {"merge_members": ["手机号"]}) + assert "合并的输入:s(节点 q_sales,数据源 merge_stores,1 行)、v(节点 q_visits,数据源 merge_members,1 行)" in text + assert "13800000000" not in text and "遮罩" in text + + async def test_a_truncated_input_fails_the_node(monkeypatch, sources): row = await run_graph(monkeypatch, store_graph(visits_args={"limit": 2}, report=False)) assert row.status == "failed" diff --git a/backend/tests/test_merge_provenance.py b/backend/tests/test_merge_provenance.py new file mode 100644 index 00000000..f90c72c2 --- /dev/null +++ b/backend/tests/test_merge_provenance.py @@ -0,0 +1,113 @@ +"""合并查询的结果接进证据下钻(期 4「推断的来源」):宁可不下钻,也不下钻错。 + +上传表格(合成客流表,按配方导入,夹具同 test_provenance_api)查一次、手工源的气温快照现造一份,两者在库外按日期 +合并。合并结果里直接来自客流表的格,下钻结论要和直接引用客流表那一格完全相同(同一个原表格子);合并 SQL 算出来的 +列判为没有逐格来历;来自手工源的格照手工源的结论(not_upload)。每份结论都断言 contract_problems 为空。 +""" +from __future__ import annotations + +import pytest + +from app.api import evidence as api +from app.core import artifact_store +from app.data.provenance_types import DOC_PROVENANCE, MergeHop +from app.engine.merge_query import MERGE_SOURCE, MergeInput, execute +from tests.fixtures.xlsx.flow import SHEET +from tests.test_provenance_api import ( # noqa: F401 夹具按名字注入(rep 依赖 upload_world、client) + cell_seg, + cell_value, + client, + ok, + put, + rep, + report_of, + run_sql, + upload_world, +) + +FLOW_SQL = 'SELECT "日期", "全日客流" FROM "日客流" ORDER BY "日期" DESC LIMIT 6' +MERGE_SQL = ('SELECT f."日期", f."全日客流", w."气温", f."全日客流" * 2 AS 加倍 FROM f JOIN w ON f."日期" = w."日期" ' + 'ORDER BY f."日期"') + + +async def merged_world(up, c): + """(文档, 封存范围, 合并结果, 客流快照)。客流按日期倒序查,合并结果按日期正序排:合并结果第 0 行是客流第 5 行。""" + qid, sid = await run_sql(c, up, FLOW_SQL) + flow = artifact_store.load(qid) + dates = sorted(r[0] for r in flow["rows"]) + weather = {"columns": ["日期", "气温"], "rows": [[d, 20 + i] for i, d in enumerate(dates)], "row_count": len(dates), + "truncated": False, "sql": "SELECT 日期, 气温 FROM 每日气温", "source": "weather_demo", + "column_types": {"日期": "text", "气温": "number"}} + wid = await put(weather) + outcome = execute([MergeInput(alias="f", node_id="q_flow", artifact=qid, snapshot=flow), + MergeInput(alias="w", node_id="q_weather", artifact=wid, snapshot=weather)], MERGE_SQL) + mid = await put(outcome.snapshot()) + doc = {"schema": "agentlab.report/1", "entity_syntax": 2, "run_id": "run-merge", "node_id": "write", + "provenance": DOC_PROVENANCE, "blocks": [], "catalog": { + "Q1": {"kind": "query", "artifact": qid, "tool": f"db_query__{up.name}", "node_id": "q_flow", + "source": up.name}, + "Q2": {"kind": "query", "artifact": wid, "tool": "db_query__weather_demo", "node_id": "q_weather", + "source": "weather_demo"}, + "Q3": {"kind": "query", "artifact": mid, "node_id": "merge", "source": MERGE_SOURCE}}} + seal = {"sealed": True, "ok": True, "manifest_seq": 9, "legacy": False} + sealed = api._Sealed(run_id="run-merge", graph={}, seal=seal, events=[], queries={ + qid: {"node_id": "q_flow", "tool": f"db_query__{up.name}"}, + wid: {"node_id": "q_weather", "tool": "db_query__weather_demo"}, + mid: {"node_id": "merge", "source": MERGE_SOURCE}}, schemas={sid: {"node_id": "q_flow"}}) + return doc, sealed, outcome, flow + + +async def drill(doc, sealed, *, alias, row, column): + return ok(await api.segment_provenance(report_of(doc), cell_seg(alias=alias, row=row, column=column), sealed, + masks=api._Masks())) + + +async def test_a_merged_upload_cell_points_at_the_same_original_cell(rep, client): + doc, sealed, outcome, flow = await merged_world(rep, client) + assert len(outcome.rows) == 6 + for r in range(len(outcome.rows)): + via_merge = await drill(doc, sealed, alias="Q3", row=r, column="全日客流") + direct = await drill(doc, sealed, alias="Q1", row=5 - r, column="全日客流") + assert via_merge.status == direct.status == "inferred" + # 同一个原表格子、同一份数据版本:结论逐字相同,只是 cell 仍是报告引用的那一格,并记下经过的合并 + assert via_merge.cell_source == direct.cell_source and via_merge.version == direct.version + assert (via_merge.cell.alias, via_merge.cell.row, via_merge.cell.column) == ("Q3", r, "全日客流") + assert via_merge.merge == [MergeHop(alias="Q3", node_id="merge", input="f", query="Q1", row=5 - r, + column="全日客流")] + # 原表那一格的值就是合并结果里这一格的值 + assert cell_value(rep.raws[0], SHEET, via_merge.cell_source.cell) == outcome.rows[r][1] + + +async def test_a_computed_merge_column_is_not_drilled(rep, client): + doc, sealed, _outcome, _flow = await merged_world(rep, client) + out = await drill(doc, sealed, alias="Q3", row=0, column="加倍") + assert (out.status, out.reason.code, out.version, out.cell_source) == ("none", "merge_no_lineage", None, None) + assert out.merge == [MergeHop(alias="Q3", node_id="merge")] + assert "表级来历" in out.reason.text + + +async def test_a_manual_source_column_keeps_its_own_answer(rep, client): + doc, sealed, _outcome, _flow = await merged_world(rep, client) + out = await drill(doc, sealed, alias="Q3", row=0, column="气温") + assert (out.status, out.reason.code) == ("none", "not_upload") + assert out.merge == [MergeHop(alias="Q3", node_id="merge", input="w", query="Q2", row=0, column="气温")] + + +async def test_an_unsealed_merge_snapshot_is_not_drilled(rep, client): + doc, sealed, _outcome, _flow = await merged_world(rep, client) + sealed.queries.pop(doc["catalog"]["Q3"]["artifact"]) + out = await drill(doc, sealed, alias="Q3", row=0, column="全日客流") + assert (out.status, out.reason.code) == ("none", "not_sealed") + + +@pytest.mark.parametrize("column, hinted", [("全日客流", True), ("气温", False), ("加倍", False)]) +async def test_the_query_step_hint_follows_the_trace(rep, client, column, hinted): + """界面只在追得到上传表格的格上请求推断来源;合并步骤后面接上追到的输入步骤。""" + doc, sealed, _outcome, _flow = await merged_world(rep, client) + steps = await api._query_steps(doc["catalog"]["Q3"]["artifact"], [(0, column)], doc, sealed, api._Masks(), + hint=True) + assert steps[0]["merge"]["inputs"][0]["query"] == "Q1" + assert ("provenance" in steps[0]) is hinted + assert len(steps) == (1 if column == "加倍" else 2) + if len(steps) == 2: + assert steps[1]["merged_into"] == "Q3" and "provenance" not in steps[1] diff --git a/backend/tests/test_provenance_contract.py b/backend/tests/test_provenance_contract.py index 730eae52..806e0953 100644 --- a/backend/tests/test_provenance_contract.py +++ b/backend/tests/test_provenance_contract.py @@ -44,7 +44,7 @@ def test_literals(): "legacy_doc", "not_cell", "not_sealed", "not_upload", "simple_upload", "manifest_unreadable", "chain_mismatch", "expression", "alias", "multi_table", "unparsed", "no_pk", "pk_missing", "masked", "null_value", "null_pk", "snapshot_gone", "db_tampered", "recheck_missing", "recheck_multiple", - "recheck_mismatch", "no_lineage"} + "recheck_mismatch", "no_lineage", "merge_no_lineage"} assert len(get_args(P.ReasonCode)) == len(set(get_args(P.ReasonCode))) assert set(get_args(P.AlertCode)) == {"db_tampered", "chain_mismatch", "manifest_unreadable"} assert set(get_args(P.AlertCode)) <= set(get_args(P.ReasonCode)) @@ -130,7 +130,10 @@ SHAPES: dict[str, list[tuple[str, object]]] = { ("cell_status", None), ("acceptance", None), ("detail", None)], "ProvenanceOut": [("schema", P.SCHEMA), ("report", REQ), ("segment", REQ), ("cell", None), ("status", REQ), ("reason", None), ("alert", None), ("sealed", REQ), ("version", None), - ("cell_source", None), ("checks", FACTORY)], + ("cell_source", None), ("checks", FACTORY), ("merge", FACTORY)], + # 合并查询结果里的格经过的每一次合并(P3:合并查询节点) + "MergeHop": [("alias", REQ), ("node_id", None), ("input", None), ("query", None), ("row", None), + ("column", None)], # 中间结构(P4-SPEC 7.2–7.4) "DirectSelect": [("table", REQ), ("alias", REQ), ("columns", REQ)], "SelectRefusal": [("code", REQ), ("detail", REQ)], @@ -146,7 +149,7 @@ SHAPES: dict[str, list[tuple[str, object]]] = { } RESPONSE = ("ReportRef", "CellRef", "Reason", "Alert", "TableRef", "PeriodView", "AcceptanceView", "StateNote", "PartView", "VersionView", "FromCell", "YearSource", "CanonicalView", "Recheck", "CellSource", - "RelatedCheck", "ProvenanceOut") + "RelatedCheck", "ProvenanceOut", "MergeHop") @pytest.mark.parametrize("name", sorted(SHAPES)) @@ -184,7 +187,8 @@ def test_intermediate_types_take_positional_args_as_the_spec_writes_them(): KEYS = { "top": ["schema", "report", "segment", "cell", "status", "reason", "alert", "sealed", "version", "cell_source", - "checks"], + "checks", "merge"], + "merge": ["alias", "node_id", "input", "query", "row", "column"], "report": ["node_id", "doc_artifact"], "cell": ["alias", "row", "column", "artifact"], "reason": ["code", "detail", "text"], @@ -248,6 +252,7 @@ def full_out() -> P.ProvenanceOut: checks=[P.RelatedCheck(id="R1", kind="relation_sum_eq", title="「日客流」:全日客流 = 分区甲 + 分区乙", part_status="mismatch", row_status="passed", cell_status=None, acceptance=_accept(), detail="合成的原因摘要")], + merge=[P.MergeHop(alias="Q3", node_id="merge", input="f", query="Q1", row=5, column="全日客流")], ) @@ -258,7 +263,7 @@ def test_asdict_matches_the_spec_json_level_by_level(): "top": d, "report": d["report"], "cell": d["cell"], "version": v, "table": v["tables"][0], "part": part, "period": part["period"], "acceptance": part["acceptances"][0], "state": part["purged"], "cell_source": cs, "from": cs["from"][0], "year": cs["year"], "canonical": cs["canonical"], - "recheck": cs["recheck"], "check": d["checks"][0], + "recheck": cs["recheck"], "check": d["checks"][0], "merge": d["merge"][0], } for name, obj in levels.items(): assert isinstance(obj, dict), name @@ -280,7 +285,8 @@ def test_null_and_empty_defaults_match_the_spec(): reason=P.refuse("legacy_doc"), sealed=False) d = asdict(out) assert d["schema"] == "agentlab.provenance/1" - assert (d["cell"], d["alert"], d["version"], d["cell_source"], d["checks"]) == (None, None, None, None, []) + assert (d["cell"], d["alert"], d["version"], d["cell_source"], d["checks"], d["merge"]) == ( + None, None, None, None, [], []) assert d["reason"] == {"code": "legacy_doc", "detail": "", "text": P.REASON_TEXT["legacy_doc"]} @@ -666,7 +672,7 @@ TS_INTERFACES = { "ProvenancePeriod": "PeriodView", "ProvenanceAcceptance": "AcceptanceView", "ProvenanceStateNote": "StateNote", "ProvenancePart": "PartView", "ProvenanceVersion": "VersionView", "ProvenanceFromCell": "FromCell", "ProvenanceYear": "YearSource", "ProvenanceCanonical": "CanonicalView", "ProvenanceRecheck": "Recheck", - "ProvenanceCellSource": "CellSource", "ProvenanceCheck": "RelatedCheck", + "ProvenanceCellSource": "CellSource", "ProvenanceCheck": "RelatedCheck", "ProvenanceMergeHop": "MergeHop", } TS_UNIONS = { "ProvenanceStatus": P.Status, "ProvenanceReasonCode": P.ReasonCode, "ProvenanceAlertCode": P.AlertCode, diff --git a/frontend/src/run/__tests__/evidence-provenance.json b/frontend/src/run/__tests__/evidence-provenance.json index 6175aed9..c8f1325f 100644 --- a/frontend/src/run/__tests__/evidence-provenance.json +++ b/frontend/src/run/__tests__/evidence-provenance.json @@ -6639,7 +6639,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s8": { "schema": "agentlab.provenance/1", @@ -6793,7 +6794,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s12": { "schema": "agentlab.provenance/1", @@ -6952,7 +6954,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s16": { "schema": "agentlab.provenance/1", @@ -7018,7 +7021,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s20": { "schema": "agentlab.provenance/1", @@ -7084,7 +7088,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s24": { "schema": "agentlab.provenance/1", @@ -7150,7 +7155,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s28": { "schema": "agentlab.provenance/1", @@ -7216,7 +7222,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s56": { "schema": "agentlab.provenance/1", @@ -7282,7 +7289,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s60": { "schema": "agentlab.provenance/1", @@ -7466,7 +7474,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s64": { "schema": "agentlab.provenance/1", @@ -7642,7 +7651,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s68": { "schema": "agentlab.provenance/1", @@ -7877,7 +7887,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s72": { "schema": "agentlab.provenance/1", @@ -8047,7 +8058,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s76": { "schema": "agentlab.provenance/1", @@ -8217,7 +8229,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s84": { "schema": "agentlab.provenance/1", @@ -8357,7 +8370,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s88": { "schema": "agentlab.provenance/1", @@ -8447,7 +8461,8 @@ "ok": true } }, - "checks": [] + "checks": [], + "merge": [] }, "s92": { "schema": "agentlab.provenance/1", @@ -8472,7 +8487,8 @@ "sealed": true, "version": null, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s52": { "schema": "agentlab.provenance/1", @@ -8626,7 +8642,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s32": { "schema": "agentlab.provenance/1", @@ -8695,7 +8712,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s36": { "schema": "agentlab.provenance/1", @@ -8764,7 +8782,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s48": { "schema": "agentlab.provenance/1", @@ -8830,7 +8849,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s44": { "schema": "agentlab.provenance/1", @@ -8858,7 +8878,8 @@ "sealed": true, "version": null, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s40": { "schema": "agentlab.provenance/1", @@ -9016,7 +9037,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s80": { "schema": "agentlab.provenance/1", @@ -9190,7 +9212,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] } }, "errors": { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index eff735f7..6ce60836 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -2262,7 +2262,7 @@ export type ProvenanceReasonCode = | 'legacy_doc' | 'not_cell' | 'not_sealed' | 'not_upload' | 'simple_upload' | 'manifest_unreadable' | 'chain_mismatch' | 'expression' | 'alias' | 'multi_table' | 'unparsed' | 'no_pk' | 'pk_missing' | 'masked' | 'null_value' | 'null_pk' | 'snapshot_gone' | 'db_tampered' | 'recheck_missing' | 'recheck_multiple' - | 'recheck_mismatch' | 'no_lineage' + | 'recheck_mismatch' | 'no_lineage' | 'merge_no_lineage' /** 标红的提示:出现时 reason.code 与它相同,界面只画提示 */ export type ProvenanceAlertCode = 'db_tampered' | 'chain_mismatch' | 'manifest_unreadable' @@ -2470,4 +2470,19 @@ export interface EvidenceProvenance { version: ProvenanceVersion | null cell_source: ProvenanceCellSource | null checks: ProvenanceCheck[] + /** 被引用的格在合并查询的结果里时,经过的每一次合并;别的查询为空 */ + merge: ProvenanceMergeHop[] +} + +/** 经过的一次合并查询:合并结果里被引用的格追到了哪个输入的哪一格;追不到的那一跳后四项为 null */ +export interface ProvenanceMergeHop { + /** 合并结果在目录里的编号(Q3) */ + alias: string + node_id: string | null + /** 追到的输入别名(合并 SQL 里的表名) */ + input: string | null + /** 那个输入在目录里的编号(Q1) */ + query: string | null + row: number | null + column: string | null }