diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 78eb92db..a51e564c 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -163,6 +163,18 @@ NODE_REFERENCE = """\ - supervisor：多 agent 协作。config: {goal, agents:[{name, description, system, tools, model}], max_rounds} 成员停不下来等人：需要审批的 MCP / 自定义工具在成员手里不会执行。要用这类工具，交给团队外的 agent 节点 - tool：直接调一个工具。config: {tool: 工具名, args: {...}, assign_to} +- merge：合并查询，把几次查询的结果在库外按键合并成一张表。config: {inputs: {别名: 上游查询节点id}, sql, assign_to} + **只在数据分在不同的库、要按行对齐时用**：先在各自的库里用 SQL 聚合到相同的键和粒度（比如「日期 + 门店」）， + 再由 merge 按键合并。同一个库里的数据写成一条 SQL（JOIN / WITH），不要拆成两次查询再合并；只组合几个单值时 + 用口径卡的 cell()，不用 merge。比率、增幅、占比这类派生计算仍然写在口径卡里，不写进合并 SQL。 + inputs 的值只能是调用 db_query__ 工具的 tool 节点或另一个 merge 节点，而且要连在 merge 之前；agent 查过的库 + 不能当输入（它可能查了好几次），要合并的那条查询单独放进 tool 节点。别名就是 sql 里的表名，只用英文字母、数字 + 和下划线（例如 s、v）。sql 是一条 SQLite 的 SELECT 或 WITH，例如：SELECT s.日期, s.门店, s.订单数, v.到店人数 + FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店。合并读的是每个输入的完整结果：任何一个输入被截断（超过 + 查询的行数上限）merge 就会失败，所以源查询要先聚合、缩小范围；两边的键类型要一致（文本 '001' 对数值 1 会给警告）； + 结果比行数最多的输入还多，说明合并键不唯一（也会给警告）。结果和查询同形：报告按先后编号引用 [[v:Qn.r0.列名]]， + 口径卡写 cell(nodes.merge节点id, 行, '列名')。选取项直接写输入的列（可以用 AS 改名）时，报告里的数点得开、能追到 + 输入的那一格；表达式算出来的列只有表级来历 - code：沙箱里跑代码。config: {language: python|bash|node, code, timeout, network, assign_to} evidence_role：source（取数：产出本身就是源数据）/ compute（计算，默认）。负责取数的代码写 source； compute 的产出喂给口径卡会被提示「核对不了出处」——业务计算写进口径卡的表达式。 @@ -194,7 +206,7 @@ NODE_REFERENCE = """\ - report：报告撰写（带引用）。config: {instructions, system, metrics_from:[口径卡id], numbers, on_violation, max_repairs, claims, entities, assign_to} **凡是给人看的、带数字的报告、分析、结论，都用 report，不要用 llm**。它自动收集上游的口径卡指标、查询结果 - （tool 节点和 agent 查过的库，按查询先后编成 Q1、Q2…）、查库当时的表结构、知识库检索（按先后编成 K1、K2…） + （tool 节点和 agent 查过的库、merge 的合并结果，按查询先后编成 Q1、Q2…）、查库当时的表结构、知识库检索（按先后编成 K1、K2…） 和运行输入，模型只能写引用标记，数值由系统从证据里取出来渲染；模型自己写的数字会被打回重写： - [[m:指标id]]：口径卡指标。比率、增幅、占比、差值这类派生计算只能先在口径卡里登记再引用 - [[v:Q1.r0.列名]]：第 1 次查询结果里第 0 行那一列的原值（行号从 0 数） @@ -234,6 +246,8 @@ NODE_REFERENCE = """\ 2. 能用 SUM / COUNT 做的聚合放在 SQL 里，口径卡只做标量运算 3. 不要用 code 节点做业务计算；code 只做格式转换（负责取数的 code 标 evidence_role: source） 4. 不要用 transform 解析 agent / llm 的文字：口径卡读 cell(nodes.查库节点id, 行, '列名') 或 vars..字段 +5. 数据在两个库、要按行对齐时：两个 tool 节点各自聚合到相同的键和粒度 → merge 按键合并 → 口径卡算派生指标 → report。 + 同一个库写成一条 SQL；只组合几个单值用口径卡的 cell()；不要用 code 节点合并（出处会断，喂给口径卡时核对不了） 模型字段（llm / agent / supervisor / report 的 config.model）： - **不要填**。留空表示跟随当前供应商的默认模型，这几乎总是对的。 @@ -1154,7 +1168,8 @@ def _evidence_source(node: Any) -> bool: if node.type == NodeType.INPUT: fields = node.config.get("fields") return isinstance(fields, list) and any(_numeric_input(f) for f in fields) - return node.type in (NodeType.METRICS, NodeType.TOOL, NodeType.RETRIEVE, NodeType.SUBGRAPH, NodeType.SUPERVISOR) + return node.type in (NodeType.METRICS, NodeType.TOOL, NodeType.MERGE, NodeType.RETRIEVE, NodeType.SUBGRAPH, + NodeType.SUPERVISOR) def _unsourced_reports(spec: GraphSpec, changed: set[str]) -> list[dict[str, Any]]: diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index 7b3ba4c7..c7623de3 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -58,7 +58,9 @@ from app.data.provenance_types import ( CellRef, Chain, ChainProblem, + REASON_TEXT, LocateProblem, + MergeHop, ProvenanceOut, RawState, Reason, @@ -100,6 +102,7 @@ from app.engine.evidence import ( uncited_claims, ) from app.engine.judge import FIELDS_MAX, SETTLED, UNSUPPORTED, VERDICTS, candidates, outdated +from app.engine.merge_query import MERGE_SOURCE, trace_cell from app.engine.toolcalls import QUERY_PREFIX router = APIRouter(prefix="/api/runs", tags=["evidence"]) @@ -426,6 +429,12 @@ def _graph_of(report: _Report, sealed: _Sealed, inputs: dict[str, Any]) -> tuple if entry.get("kind") in ("query", "retrieval"): # 查询、检索条目带上是什么、有多大：图上不点开也看得出这是哪次取数 item.update({k: entry[k] for k in ("tool", "source", "columns", "rows", "truncated") if k in entry}) + if entry.get("kind") == "query" and entry.get("source") == MERGE_SOURCE: + # 合并结果连向它合并的那几次查询：图上看得出 Q3 是 Q1、Q2 按键合并出来的 + edges.extend({"from": alias, "to": target, "rel": "merged_from", "input": source.get("alias"), + "report": report.node_id} + for source in _merge_inputs(entry.get("artifact")) + if (target := _query_alias(catalog, source.get("artifact")))) elif entry.get("kind") in ENTITY_KINDS: # 表和字段：叫什么、出现在哪几次查询里、来历（表结构快照 / SQL / 结果列）各自封存了没有 item.update({k: entry[k] for k in ("name", "table", "tables", "qualified", "source") if entry.get(k)}) @@ -454,6 +463,16 @@ def _graph_of(report: _Report, sealed: _Sealed, inputs: dict[str, Any]) -> tuple return evidence, edges +def _merge_inputs(artifact: Any) -> list[dict[str, Any]]: + """合并结果快照记下的输入；取不回、哈希不符的当没有（证据图只是少几条边，片段接口照样说清楚）。""" + try: + snap = load(str(artifact)) if artifact else None + except ValueError: + return [] + inputs = snap.get("inputs") if isinstance(snap, dict) else None + return [i for i in inputs if isinstance(i, dict)] if isinstance(inputs, list) else [] + + def _cell_name(alias: Any, locator: dict[str, Any]) -> str: """Q1.r0.gmv：报告目录里的全局编号加行列，和 [[v:]] 的写法一致。""" if not alias or not isinstance(locator.get("row"), int) or not locator.get("column"): @@ -778,8 +797,8 @@ async def _chain(seg: dict[str, Any], doc: dict[str, Any], sealed: _Sealed, locator = cite.get("locator") or {} # 只有直接引用的单元格才可能推断来源（P4-SPEC 2.8.2）。指标链里的查询步骤不带：指标片段的推断来源只会 # 回「不是单元格」，带上提示等于让界面白发一次请求 - return [await _query_step(str(entry.get("artifact") or ""), [(locator.get("row"), locator.get("column"))], - doc, sealed, masks, hint=doc.get("provenance") == DOC_PROVENANCE)] + return await _query_steps(str(entry.get("artifact") or ""), [(locator.get("row"), locator.get("column"))], + doc, sealed, masks, hint=doc.get("provenance") == DOC_PROVENANCE) if cite.get("kind") == "input": name = (cite.get("locator") or {}).get("field") inputs = _input_payloads(sealed) @@ -846,7 +865,7 @@ async def _metric_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[s one["cell"] = cell wanted.setdefault(str(artifact_of), []).extend(_located(item.get("locator") or {})) inputs.append(one) - queries = [await _query_step(a, cells, doc, sealed, masks) for a, cells in wanted.items()] + queries = [s for a, cells in wanted.items() for s in await _query_steps(a, cells, doc, sealed, masks)] return [step, *inputs, *queries] @@ -935,20 +954,30 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str columns = [str(c) for c in snap.get("columns") or []] rows = snap["rows"] step["source"] = step["source"] or snap.get("source") - # 遮的是「查询当时记下的」和「数据源现在设的」两者之和：事后加的遮罩照样生效，数据源改名、 - # 删掉了也不会把当时遮着的列亮出来 - hidden, found = await masks.of(step["source"]) - recorded = snap.get("mask_columns") - hidden = hidden | ({str(c).lower() for c in recorded if isinstance(c, str)} if isinstance(recorded, list) else set()) - if not found: - step["mask_note"] = SOURCE_GONE.format(source=step["source"]) + merged = snap.get("source") == MERGE_SOURCE + if merged: + # 合并查询不是数据源：遮罩按它合并的那几个数据源算（_merge_masks），不能拿「合并查询」去找数据源， + # 找不到再说它「已不存在」 + hidden, gone = await _merge_masks(snap, masks) + if gone: + step["mask_note"] = SOURCE_GONE.format(source="、".join(gone)) + step["merge"] = _merge_view(snap, cells, catalog, sealed) + if hint and _leads_to_upload(snap, cells): + step["provenance"] = True + else: + # 遮的是「查询当时记下的」和「数据源现在设的」两者之和：事后加的遮罩照样生效，数据源改名、 + # 删掉了也不会把当时遮着的列亮出来 + hidden, found = await masks.of(step["source"]) + hidden = hidden | _recorded_masks(snap) + if not found: + step["mask_note"] = SOURCE_GONE.format(source=step["source"]) masked = {i for i, c in enumerate(columns) if c.lower() in hidden} index = sorted({i for r in rows_hit for i in range(r - WINDOW, r + WINDOW + 1) if 0 <= i < len(rows)}) step.update(sql=snap.get("sql"), columns=columns, total_rows=len(rows), truncated=bool(snap.get("truncated")), masked=[columns[i] for i in sorted(masked)], # 查询时从驱动的原始值记下的列类型（老快照没有）：文本列里的 "2026" 按文本显示，不当数 column_types=snap.get("column_types") if isinstance(snap.get("column_types"), dict) else {}, - **({"provenance": True} if hint and snap.get("data_version") else {})) + **({"provenance": True} if hint and snap.get("data_version") and not merged else {})) if len(index) > MAX_WINDOW_ROWS: index, step["window_truncated"] = index[:MAX_WINDOW_ROWS], True step["row_index"] = index @@ -958,6 +987,116 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str return step +# -------------------------------------------------------------------------- +# 合并查询的结果：表级来历（输入、合并 SQL）和逐格来历（被引用的格追到输入的哪一格） +# -------------------------------------------------------------------------- + +#: 一条链最多追几层合并（合并的合并）。画布上不会无限嵌套，这只是防御坏快照 +MERGE_DEPTH = 4 + + +async def _query_steps(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str, Any], sealed: _Sealed, + masks: _Masks, *, hint: bool = False, depth: int = 0) -> list[dict[str, Any]]: + """查询步骤；快照是合并结果时，后面接上被引用的格追到的那几个输入的查询步骤（高亮追到的那一格）。 + + 追不到的格不接任何输入步骤：合并步骤的 merge.traced 里照实写「没有逐格来历」，只给表级来历。输入步骤带 + merged_into（合并结果的编号），界面据此标明它是哪次合并的输入。不是合并结果的快照，返回的就是原来那一个步骤。 + """ + step = await _query_step(artifact, cells, doc, sealed, masks, hint=hint) + traced = [t for t in (step.get("merge") or {}).get("traced") or [] if t.get("input")] + if not traced or depth >= MERGE_DEPTH: + return [step] + by_input: dict[str, list[tuple[Any, Any]]] = {} + artifacts = {str(i.get("alias")): str(i.get("artifact") or "") for i in step["merge"]["inputs"]} + for t in traced: + by_input.setdefault(artifacts.get(str(t["input"]), ""), []).append((t["row"], t["column"])) + out = [step] + for source, picked in by_input.items(): + if not source: + continue + for sub in await _query_steps(source, picked, doc, sealed, masks, depth=depth + 1): + out.append({**sub, "merged_into": sub.get("merged_into") or step.get("alias")}) + return out + + +def _merge_view(snap: dict[str, Any], cells: list[tuple[Any, Any]], catalog: dict[str, Any], + sealed: _Sealed) -> dict[str, Any]: + """合并步骤的 merge 一节：每个输入（别名、节点、节点名、目录编号、行数、数据源、封存与否）、合并 SQL、执行时的 + 警告，以及被引用的每一格追到了哪个输入的哪一格（追不到的写明原因）。""" + labels = {str(n.get("id")): (n.get("data") or {}).get("label") or None + for n in sealed.graph.get("nodes") or [] if isinstance(n, dict)} + inputs = [] + for item in snap.get("inputs") or []: + if not isinstance(item, dict): + continue + art = str(item.get("artifact") or "") + inputs.append({"alias": item.get("alias"), "node_id": item.get("node_id"), + "label": labels.get(str(item.get("node_id"))), "query": _query_alias(catalog, art), + "rows": item.get("rows"), "source": item.get("source"), "artifact": art or None, + "sealed": sealed.trusted and art in sealed.queries}) + traced = [] + for row, column in dict.fromkeys((r, c) for r, c in cells if isinstance(r, int) and c): + hit = trace_cell(snap, row, column) + if hit is None: + traced.append({"cell": [row, str(column)], "input": None, "query": None, "row": None, "column": None, + "note": REASON_TEXT["merge_no_lineage"]}) + else: + traced.append({"cell": [row, str(column)], "input": hit.alias, "query": _query_alias(catalog, hit.artifact), + "row": hit.row, "column": hit.column}) + warnings = [{"code": w.get("code"), "message": w.get("message")} for w in snap.get("warnings") or [] + if isinstance(w, dict) and w.get("message")] + return {"sql": snap.get("sql"), "inputs": inputs, "warnings": warnings, "traced": traced} + + +async def _merge_masks(snap: dict[str, Any], masks: _Masks) -> tuple[set[str], list[str]]: + """合并结果要遮的列（小写）和已经不在的数据源： + + - 合并当时记下的（快照的 mask_columns：各输入当时的遮罩列，加上来自遮罩列、换了名字的结果列）； + - 它合并的每个数据源现在设的（按列名）； + - 结果列有逐格来历、来自某个数据源现在遮着的列的，按来历遮（换了名字也遮住）。宁可多遮。""" + hidden = _recorded_masks(snap) + gone: list[str] = [] + for name in snap.get("sources") or []: + cols, found = await masks.of(name) + hidden |= cols + if not found: + gone.append(str(name)) + lineage = snap.get("lineage") if isinstance(snap.get("lineage"), dict) else {} + origins = lineage.get("sources") or [] + by_alias = {str(i.get("alias")): i for i in snap.get("inputs") or [] if isinstance(i, dict)} + for name, target in zip(snap.get("columns") or [], lineage.get("columns") or []): + if not isinstance(target, dict) or not isinstance(target.get("source"), int) \ + or not 0 <= target["source"] < len(origins): + continue + origin = origins[target["source"]] if isinstance(origins[target["source"]], dict) else {} + source = (by_alias.get(str(origin.get("input"))) or {}).get("source") + if source and source != MERGE_SOURCE and str(target.get("column") or "").lower() in (await masks.of(source))[0]: + hidden.add(str(name).lower()) + return hidden, gone + + +def _leads_to_upload(snap: dict[str, Any], cells: list[tuple[Any, Any]], depth: int = 0) -> bool: + """被引用的格经合并追下去，最后落在上传表格的查询快照上（有 data_version）：界面才去请求推断来源。""" + if depth >= MERGE_DEPTH: + return False + for row, column in cells: + hit = trace_cell(snap, row, column) + if hit is None or not hit.artifact: + continue + try: + source = load(hit.artifact) + except ValueError: + continue + if not isinstance(source, dict): + continue + if source.get("source") == MERGE_SOURCE: + if _leads_to_upload(source, [(hit.row, hit.column)], depth + 1): + return True + elif source.get("data_version"): + return True + return False + + # -------------------------------------------------------------------------- # GET /api/runs/{run_id}/evidence/segments/{segment_id}/provenance（期 4：推断的来源） # -------------------------------------------------------------------------- @@ -1073,6 +1212,10 @@ async def segment_provenance(report: _Report, seg: dict[str, Any], sealed: _Seal # R0：期 4 之前组装的文档（没有标记）一律不推断，期 4 之前的面板不变（1.4） if doc.get("provenance") != DOC_PROVENANCE: return none("legacy_doc") + # 合并查询结果里的格：先按逐格来历追到输入的那一格，再对那一格照常判断；追不到就只有表级来历 + if (cite.get("kind") == "cell" and cite.get("status") == "resolved" and seg.get("state") == "deterministic" + and entry.get("kind") == "query" and entry.get("source") == MERGE_SOURCE): + return await _merge_provenance(report, seg, entry, sealed, masks=masks, base=base) # R1：只对直接引用的查询单元格下钻；代码节点、Agent 字段的取数（工具名不是 db_query__ 开头）不算 if (cite.get("kind") != "cell" or cite.get("status") != "resolved" or seg.get("state") != "deterministic" or entry.get("kind") != "query" or not str(entry.get("tool") or "").startswith(QUERY_PREFIX)): @@ -1097,6 +1240,46 @@ async def segment_provenance(report: _Report, seg: dict[str, Any], sealed: _Seal masks=masks, sealed=sealed) +async def _merge_provenance(report: _Report, seg: dict[str, Any], entry: dict[str, Any], sealed: _Sealed, *, + masks: _Masks, base: dict[str, Any], hops: tuple[MergeHop, ...] = ()) -> ProvenanceOut: + """合并结果里的一格：沿合并快照记下的逐格来历（engine/merge_query.py）追到输入的那一格，换成对那一格的判断。 + + 宁可不下钻，也不下钻错：追不到（计算、聚合出来的列，合并 SQL 超出可追溯的写法，一模一样的两行）就判 + merge_no_lineage，只有表级来历（合并的输入、合并 SQL 在查询步骤的 merge 一节里）。追到的输入是数据源查询时， + 结论就是直接引用那一格时的结论（上传表格照常推断原表格子，手工源 not_upload）；输入本身也是合并结果时再追一跳。 + cell 始终是报告引用的那一格，merge 列出经过的每一跳。 + """ + def out(status: str, reason: Reason, *more: MergeHop) -> ProvenanceOut: + return ProvenanceOut(status=status, reason=reason, merge=[*hops, *more], **base) # type: ignore[arg-type] + + cite = seg.get("cite") or {} + artifact = str(entry.get("artifact") or "") + snap, ok = await _Artifacts().get(artifact) if artifact in sealed.queries else (None, None) + if ok is not True or not isinstance(snap, dict): + return out("none", refuse("not_sealed")) + locator = cite.get("locator") or {} + row, column = locator.get("row"), locator.get("column") + if isinstance(column, str) and column.lower() in _recorded_masks(snap): + return out("none", refuse("masked")) + hit = trace_cell(snap, row, column) + catalog = (report.doc or {}).get("catalog") or {} + alias = _query_alias(catalog, hit.artifact) if hit is not None else None + hop = MergeHop(alias=str(cite.get("alias") or ""), node_id=entry.get("node_id"), + input=hit.alias if hit else None, query=alias, row=hit.row if hit else None, + column=hit.column if hit else None) + if hit is None or len(hops) >= MERGE_DEPTH: + return out("none", refuse("merge_no_lineage"), replace(hop, input=None, query=None, row=None, column=None)) + if alias is None: + # 输入不在这份报告的目录里（不在报告的上游、没有记进台账）：认不出它是封存过的查询 + return out("none", refuse("not_sealed"), hop) + sub_entry = catalog[alias] + sub_seg = {**seg, "cite": {**cite, "alias": alias, "locator": {"row": hit.row, "column": hit.column}}} + if sub_entry.get("source") == MERGE_SOURCE: + return await _merge_provenance(report, sub_seg, sub_entry, sealed, masks=masks, base=base, hops=(*hops, hop)) + found = await segment_provenance(report, sub_seg, sealed, masks=masks) + return replace(found, cell=base["cell"], segment=base["segment"], merge=[*hops, hop]) + + async def _drill(base: dict[str, Any], *, seg: dict[str, Any], entry: dict[str, Any], query: dict[str, Any], schema: dict[str, Any], artifacts: _Artifacts, masks: _Masks, sealed: _Sealed) -> ProvenanceOut: """R5–R17（P4-SPEC 2.3）：第一条不满足的就是结论。 diff --git a/backend/app/core/events.py b/backend/app/core/events.py index 81ed9c40..abef3e77 100644 --- a/backend/app/core/events.py +++ b/backend/app/core/events.py @@ -48,6 +48,9 @@ class EventType(StrEnum): # 记忆读写。以前发的是 info 日志，而解码器丢弃所有 info——于是往长期记忆里 # 写东西这件事在界面上是隐形的。让 Copilot 主动记之后，这条不可接受 MEMORY_END = "memory.end" + # 合并查询做完了：哪几个输入（别名、节点、行数、数据源）、哪条合并 SQL、得到几行、有哪些警告。 + # 运行详情据此展示这一步；完整结果在 query_artifact 指向的查询快照里 + MERGE_END = "merge.end" SANDBOX_START = "sandbox.start" SANDBOX_END = "sandbox.end" diff --git a/backend/app/data/engine.py b/backend/app/data/engine.py index 28439565..04a0f6eb 100644 --- a/backend/app/data/engine.py +++ b/backend/app/data/engine.py @@ -280,6 +280,15 @@ def open_checked_sqlite(path: str, *, readonly: bool = True, immutable: bool = F factory=_DqsOffConnection) +def open_memory_sqlite() -> sqlite3.Connection: + """一个内存 SQLite 连接，DQS 已关（合并查询把几次查询的结果落进来再合并，engine/merge_query.py）。 + + 合并 SQL 同样是模型写的：双引号里的列名写错一个字，默认连接会把它当成字符串常量，`SUM("销售颔")` 得 0， + `ON s."门店" = v."门店 "` 一行都对不上却不报错。所以和连接层共用 _DqsOffConnection，关不掉就不交出连接。 + """ + return sqlite3.connect(":memory:", factory=_DqsOffConnection) + + def _sqlite_extra() -> dict[str, Any]: """每个 SQLite engine 都带的连接参数：关 DQS。""" return {"connect_args": {"factory": _DqsOffConnection}} diff --git a/backend/app/data/provenance_types.py b/backend/app/data/provenance_types.py index 703d49d6..b76802df 100644 --- a/backend/app/data/provenance_types.py +++ b/backend/app/data/provenance_types.py @@ -95,6 +95,8 @@ ReasonCode = Literal[ "recheck_missing", "recheck_multiple", "recheck_mismatch", # R17 "no_lineage", + # 合并查询的结果：这一格说不清来自哪个输入的哪一格（计算、聚合出来的列，或合并 SQL 超出可追溯的写法） + "merge_no_lineage", ] #: 标红的提示。出现时 reason.code 与它相同，界面只画提示、不再画 reason.text（P4-SPEC 4.1、4.2 末尾） @@ -179,6 +181,8 @@ REASON_TEXT: dict[str, str] = { "recheck_multiple": _RECHECK_TEXT, "recheck_mismatch": _RECHECK_TEXT, "no_lineage": "导入清单里没有这一格的溯源记录，只给出表级来历", + "merge_no_lineage": ("这一格来自合并查询，无法确定它对应哪个输入的哪一格（计算、聚合出来的列，或合并 SQL 超出可追溯的" + "写法），没有逐格来历，只有表级来历：合并了哪几个输入、用的哪条合并 SQL"), } #: 标红提示的原文：与同名 reason 的原文相同（两处同时出现时界面只画提示） @@ -506,6 +510,27 @@ class RelatedCheck: detail: str | None = None +@dataclass(kw_only=True) +class MergeHop: + """经过的一次合并查询（engine/merge_query.py）：被引用的格在合并结果里，追到了哪个输入的哪一格。 + + 追不到的那一跳 input、query、row、column 都是 None，结论是 merge_no_lineage。输入本身也是合并结果时再追一跳， + ProvenanceOut.merge 按从报告引用的那一份往下的顺序列出每一跳。""" + + #: 合并结果在目录里的编号（Q3） + alias: str + #: 合并查询节点 + node_id: str | None = None + #: 追到的输入别名（合并 SQL 里的表名） + input: str | None = None + #: 那个输入在目录里的编号（Q1）；输入不在这份报告的目录里时为 None + query: str | None = None + #: 输入快照里的行号，从 0 数 + row: int | None = None + #: 输入快照里的列名 + column: str | None = None + + @dataclass(kw_only=True) class ProvenanceOut: """推断来源接口的响应。asdict 之后就是 P4-SPEC 2.8.1 的 JSON；不变式见 contract_problems。""" @@ -523,6 +548,9 @@ class ProvenanceOut: version: VersionView | None = None cell_source: CellSource | None = None checks: list[RelatedCheck] = field(default_factory=list) + #: 被引用的格在合并查询的结果里时，经过的每一次合并（见 MergeHop）；别的查询为空。下钻的结论（status、version、 + #: cell_source）说的是最后追到的那个输入的那一格，cell 仍是报告引用的那一格 + merge: list[MergeHop] = field(default_factory=list) # ========================================================================== @@ -727,4 +755,11 @@ def contract_problems(out: ProvenanceOut) -> list[str]: bad("table_only 时 version 必须有") if out.status == "none" and out.version is not None: bad("none 时 version 为 null") + if out.reason is not None and out.reason.code == "merge_no_lineage": + if out.status != "none": + bad("merge_no_lineage 时 status 为 none") + if not out.merge or out.merge[-1].input is not None: + bad("merge_no_lineage 时 merge 的最后一跳没有追到输入（input 为 null）") + if any(hop.input is None for hop in out.merge[:-1]): + bad("merge 里只有最后一跳可以没追到输入") return out_problems diff --git a/backend/app/engine/compiler.py b/backend/app/engine/compiler.py index 07c70658..05ef4472 100644 --- a/backend/app/engine/compiler.py +++ b/backend/app/engine/compiler.py @@ -11,7 +11,7 @@ from langgraph.graph import END, START, StateGraph from app.core.errors import describe_exception, raw_detail from app.core.events import EventType from app.engine.context import NodeContext, NodeError, RunContext -from app.engine.nodes import control, human, io, knowledge, llm, metrics, multi, report, tools +from app.engine.nodes import control, human, io, knowledge, llm, merge, metrics, multi, report, tools from app.engine.schema import GraphNode, GraphSpec, NodeType, back_edges, innermost_loops from app.engine.state import GraphState @@ -28,6 +28,7 @@ RUNNERS: dict[NodeType, NodeRunner] = { NodeType.SUBGRAPH: multi.run_subgraph, NodeType.TOOL: tools.run_tool, NodeType.CODE: tools.run_code, + NodeType.MERGE: merge.run_merge, NodeType.BRANCH: control.run_branch, NodeType.LOOP: control.run_loop, NodeType.HUMAN: human.run_human, diff --git a/backend/app/engine/governance.py b/backend/app/engine/governance.py index 9bdd07d3..1b07186d 100644 --- a/backend/app/engine/governance.py +++ b/backend/app/engine/governance.py @@ -237,7 +237,7 @@ REPORT_POLICY_ALSO: dict[str, tuple[str, ...]] = {"claims": (CLAIMS_JUDGE,)} JUDGE_UNLIMITED = "不设上限，费用只受句数、时长上限和每日上限约束" #: 产出本身就是证据、或者由系统确定地算出来的节点。成果字段取它们的值，不算「别的节点写的字」 -_TRACEABLE = (NodeType.REPORT, NodeType.METRICS, NodeType.INPUT, NodeType.TOOL, NodeType.RETRIEVE) +_TRACEABLE = (NodeType.REPORT, NodeType.METRICS, NodeType.INPUT, NodeType.TOOL, NodeType.RETRIEVE, NodeType.MERGE) _WRITERS = (NodeType.LLM, NodeType.AGENT, NodeType.SUPERVISOR) #: 往对话里写消息的节点（emit_message 没关时），「最后一条消息」就是它们里最后跑的那个写的 _EMITTERS = (*_WRITERS, NodeType.REPORT) diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index b505d883..8e8d67a8 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -65,6 +65,7 @@ from app.engine.evidence import ( table_fields, ) from app.engine.expressions import CellError, column_kind, locate_cell +from app.engine.merge_query import MERGE_SOURCE from app.engine.nodes.llm import _split_structured, _usage_of from app.engine.state import message_text from app.providers import catalog as pricing @@ -314,10 +315,13 @@ class _Fetch: def _hidden(snapshot: dict[str, Any], source: Any, masked: dict[str, Any] | None) -> set[str]: - """要遮住的列（小写）：查询当时记下的，加上数据源现在设的。""" + """要遮住的列（小写）：查询当时记下的，加上数据源现在设的。合并查询的结果不是数据源，按它合并的那几个数据源算。""" names = _mask_names(snapshot.get("mask_columns")) + wanted = {str(source or "")} + if snapshot.get("source") == MERGE_SOURCE and isinstance(snapshot.get("sources"), list): + wanted |= {str(s) for s in snapshot["sources"]} for name, cols in (masked or {}).items(): - if str(name) == str(source or ""): + if str(name) in wanted: names |= _mask_names(cols) return names @@ -346,9 +350,12 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter total = len(snapshot["rows"]) # 旧形状的台账条目没记数据源：按快照自己记的找遮罩（证据面板同一个口径） source = entry.get("source") or snapshot.get("source") - head = f"【{alias}】查询结果（{entry.get('tool') or '查询'} · 数据源 {source or '—'} · 共 {total} 行）" + head = (f"【{alias}】合并查询结果（共 {total} 行）" if snapshot.get("source") == MERGE_SOURCE + else f"【{alias}】查询结果（{entry.get('tool') or '查询'} · 数据源 {source or '—'} · 共 {total} 行）") sql = _clean(str(snapshot.get("sql") or "")) lines = [head, f"SQL：{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"] + if snapshot.get("source") == MERGE_SOURCE: + lines.append(_merge_line(snapshot)) hidden = _hidden(snapshot, source, masked) if provenance_on and snapshot.get("data_version"): lines += _provenance_lines(snapshot, fetch, hidden) @@ -374,6 +381,16 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter return "\n".join(lines) +def _merge_line(snapshot: dict[str, Any]) -> str: + """合并查询结果的摘录多一行：合并了哪几个输入（SQL 里的别名、节点、数据源、行数）。上面的 SQL 就是合并 SQL。""" + parts = [] + for item in snapshot.get("inputs") or []: + if isinstance(item, dict): + parts.append(f"{item.get('alias')}（节点 {item.get('node_id') or '—'}，数据源 {item.get('source') or '—'}，" + f"{item.get('rows')} 行）") + return "合并的输入：" + ("、".join(parts) if parts else "—") + + def _provenance_lines(snapshot: dict[str, Any], fetch: _Fetch, hidden: set[str]) -> list[str]: """上传表格的查询快照的来历行（P4-SPEC 3.2、3.3）：provenance.judge_lines 生成，这里只负责取冻结的表结构快照、 算 SQL 用到的表（direct_select.tables_in，和执行这条 SQL 的 SQLite 同一种读法，宁可多认），把现成的遮罩交过去。 diff --git a/backend/app/engine/labels.py b/backend/app/engine/labels.py index e16ead9f..073934e2 100644 --- a/backend/app/engine/labels.py +++ b/backend/app/engine/labels.py @@ -18,7 +18,7 @@ from typing import Any #: 节点类型的叫法，和前端 lib/terms.ts 的 NODE_TYPE_LABEL 一致 TYPE_LABEL: dict[str, str] = { "input": "输入", "output": "成果", "llm": "模型调用", "agent": "Agent", - "supervisor": "多 Agent 协作", "tool": "调用工具", "code": "沙箱代码", + "supervisor": "多 Agent 协作", "tool": "调用工具", "code": "沙箱代码", "merge": "合并查询", "branch": "条件分支", "loop": "循环", "subgraph": "子工作流", "memory": "长期记忆", "retrieve": "知识检索", "transform": "数据整形", "human": "人工审批", "validate": "结构校验", "metrics": "口径卡", "report": "报告撰写", @@ -79,6 +79,7 @@ _FIELD_BY_TYPE: dict[str, dict[str, str]] = { "human": {"mode": "审批方式", "title": "标题", "message": "展示给审批人的内容", "draft": "草稿内容", "stop_on_reject": "驳回即终止运行"}, "validate": {"source": "待校验内容", "fail_fast": "校验失败即中断"}, + "merge": {"inputs": "输入", "sql": "合并 SQL"}, } #: 出具契约里的键（canvas/issues.ts 的 CONTRACT_KEYS、SUB_KEYS） diff --git a/backend/app/engine/merge_query.py b/backend/app/engine/merge_query.py new file mode 100644 index 00000000..4d473886 --- /dev/null +++ b/backend/app/engine/merge_query.py @@ -0,0 +1,879 @@ +"""合并查询（merge 节点）：几次查询的完整结果落进内存 SQLite，再用一条 SELECT / WITH 按键合并。 + +**为什么要有它。** 跨查询组合原来只有三种：口径卡用 cell() 组合几个单值；tool 节点的 SQL 里写 {{ vars.x }}，把前一条 +查询的结果代进后一条；同一个库里写一条 JOIN / WITH。按行合并两次查询，尤其数据在两个库里时，没有路可走：用沙箱代码 +合并，出处链断在代码节点上，喂给口径卡时只能提示「核对不了出处」；让模型把 id 从一条查询抄进下一条，会抄错，结果被 +截断后还会漏。 + +**为什么不做跨库联邦查询。** 联邦查询要把两边的明细拉到一处，数据量、方言、权限一样都控制不住。方案是先在各自的库里 +聚合到共同的键和粒度（日期 + 门店），再由这里在库外按键合并：两边进来的都是已经聚合过的小结果。 + +**为什么落成 SQLite、用标准库 sqlite3。** 照搬 data/tabular.py：SQLite 是现成的 SQL 引擎，守卫有它的读法（guard +的 _SQLITE），证据下钻的识别器（direct_select）也按它分词，不必为这一步引入 DuckDB 再多一种方言。连接同样关掉 DQS +（data.engine.open_memory_sqlite）：双引号里的列名写错要报错，不能当成字符串常量。 + +**守卫和三道防护。** +- SQL 先过守卫（SQLite 读法），只认 SELECT 和 WITH；表落完之后连接设成 query_only，再挂授权回调，只放行读表、 + 调函数、递归公用表表达式。守卫是第一道，后两道不赌守卫没有漏洞； +- 结果的行数、字节上限和查询层相同（QueryLimits），超时按查询层的缺省时限由进度回调中断； +- 任一输入被截断 → 拒绝合并：拿只有前若干行的结果按键合并，缺的行不报错，只会悄悄少算； +- 合并键两边类型不一致（文本 '001' 对数值 1）→ 警告。SQLite 比较时按亲和性隐式转换，'001'、'01' 都会等于 1； +- 结果行数多于行数最多的输入 → 警告合并键可能不唯一（多对多匹配把行放大了）。 + +**逐格来历：宁可不下钻，也不下钻错。** 报告里引用合并结果的一格，证据面板要能指回它来自哪个输入的哪一格。只有一种 +写法认得出来：单层 SELECT，FROM 后面是输入表（可加别名、可 JOIN），选取项是某张输入表的列（可改名）；没有分组、 +聚合、DISTINCT、子查询、公用表表达式、集合运算、窗口和 NATURAL JOIN。认出来以后，在原 SQL 的选取列表末尾补上每张 +输入表的 rowid 再执行一次：补列的结果去掉补的几列，必须和原结果逐行逐值相同，才把 rowid 记成来历；对不上整份不记。 +另外两种情况也不给：结果里一模一样的两行（说不清哪一行来自哪一行输入），以及来历那一格和结果的值对不上的列。 +表达式、聚合出来的列没有逐格来历，只有表级来历：来自哪几个输入、哪条合并 SQL。 + +纯函数：不连数据库、不读写工件、不发事件。取上游快照、存快照、发事件在执行器（engine/nodes/merge.py）。 +""" +from __future__ import annotations + +import json +import re +import sqlite3 +import time +from collections import Counter +from dataclasses import dataclass, field +from typing import Any + +from app.data import guard +from app.data.engine import _jsonable, column_types, open_memory_sqlite +from app.data.guard import QueryLimits, SqlRejected +from app.data.names import name_key +from app.engine.direct_select import _kw, _lex, _Lex +from app.engine.expressions import same_value + +#: 合并查询结果的查询快照里 source 记这个。数据源名只能是小写英文、数字和下划线（api/datasources.py 的 +#: _NAME_PATTERN），永远撞不上它；证据面板、裁判摘录据此认出合并结果，不去按数据源名找遮罩 +MERGE_SOURCE = "合并查询" +#: 一个合并节点最多几个输入。再多通常说明该在源库里先合并 +MAX_INPUTS = 8 +#: 别名（也就是内存库里的表名）最长几个字符 +ALIAS_MAX = 32 + +_ALIAS = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +#: SQLite 的全部关键字（sqlite.org/lang_keywords.html），加上 true / false。用作表名要加引号才读得对， +#: 模型写合并 SQL 时多半不加——不如在起名时就挡住 +_SQLITE_KEYWORDS = frozenset(""" +abort action add after all alter always analyze and as asc attach autoincrement before begin between by cascade case +cast check collate column commit conflict constraint create cross current current_date current_time current_timestamp +database default deferrable deferred delete desc detach distinct do drop each else end escape except exclude exclusive +exists explain fail filter first following for foreign from full generated glob group groups having if ignore immediate +in index indexed initially inner insert instead intersect into is isnull join key last left like limit match +materialized natural no not nothing notnull null nulls of offset on or order others outer over partition plan pragma +preceding primary query raise range recursive references regexp reindex release rename replace restrict returning +right rollback row rows savepoint select set table temp temporary then ties to transaction trigger unbounded union +unique update using vacuum values view virtual when where window with without true false +""".split()) + +#: 快照里记的列类型 → 内存表的列亲和性。number 用 NUMERIC：数据源把 DECIMAL 落成的文本（"3200.50"）进来就是数， +#: 能直接参与运算；文本、日期一律 TEXT，'007' 的前导零不丢。拿不准的列不写类型，值原样存 +_AFFINITY = {"number": "NUMERIC", "boolean": "INTEGER", "text": "TEXT", "date": "TEXT", "datetime": "TEXT", + "time": "TEXT"} +_KIND_LABEL = {"number": "数值", "boolean": "布尔值", "text": "文本", "date": "日期", "datetime": "日期时间", + "time": "时间"} +#: SQLite 表的 rowid 有三个名字；输入里恰好有同名的列时换下一个，三个都占了就不追逐格来历 +_ROWID_NAMES = ("rowid", "_rowid_", "oid") +#: 补进选取列表的来历列。带引号写，和输入的列名撞不上（输入的列名里不会有这个前缀，撞了也只是对不上、不记来历） +_LINEAGE_COLUMN = "__agentlab_merge_row_{}" + +SELECT_ONLY = "合并 SQL 只能是 SELECT 或 WITH 查询" +_TEMPLATE = re.compile(r"\{\{.*?\}\}", re.S) + +#: 授权回调放行的动作：读表、调函数、递归公用表表达式。别的一律拒（ATTACH、PRAGMA、写） +_ALLOWED_ACTIONS = frozenset({sqlite3.SQLITE_SELECT, sqlite3.SQLITE_READ, sqlite3.SQLITE_FUNCTION, + sqlite3.SQLITE_RECURSIVE}) + + +class MergeError(ValueError): + """合并做不成。消息直接给人看：哪个输入、为什么、怎么改。""" + + +@dataclass(frozen=True) +class MergeInput: + """一个输入：别名（内存库里的表名）、上游节点、它的查询快照。""" + + alias: str + node_id: str + #: 上游查询快照的工件 id（tool 节点的 query_snapshot，或另一个合并节点的结果快照） + artifact: str + snapshot: dict[str, Any] + #: 上游节点在画布上的名字，只用在报错里 + label: str = "" + + +@dataclass(frozen=True) +class Trace: + """合并结果的一格来自哪个输入的哪一格（只走一跳；输入本身是合并结果时由调用方再追）。""" + + alias: str + node_id: str | None + artifact: str | None + row: int + column: str + + +@dataclass +class MergeOutcome: + columns: list[str] + rows: list[list[Any]] + truncated: bool + #: 守卫规范过的那一条 SQL，执行的就是它 + sql: str + column_types: dict[str, str] + warnings: list[dict[str, Any]] + #: 逐格来历（见模块说明）；认不出写法、核对不上时为 None + lineage: dict[str, Any] | None + inputs: list[dict[str, Any]] + #: 底层数据源（输入本身是合并结果的，展开成它的数据源），去重保序 + sources: list[str] + mask_columns: list[str] = field(default_factory=list) + + def snapshot(self) -> dict[str, Any]: + """存进工件库的查询快照：和数据源查询同形（columns、rows、row_count、truncated、sql、column_types、source）， + 另记 inputs、sources、warnings、lineage。不含耗时之类每次不同的东西：同样的输入、同样的 SQL 得到同一个工件， + 节点重放、继续运行时快照 id 不变。""" + out: dict[str, Any] = { + "columns": self.columns, "rows": self.rows, "row_count": len(self.rows), "truncated": self.truncated, + "sql": self.sql, "source": MERGE_SOURCE, "inputs": self.inputs, "sources": self.sources, + "warnings": self.warnings, + } + if self.column_types: + out["column_types"] = self.column_types + if self.lineage is not None: + out["lineage"] = self.lineage + if self.mask_columns: + out["mask_columns"] = self.mask_columns + return out + + +# -------------------------------------------------------------------------- +# 静态检查：别名、SQL（画图时校验用，执行时同一套规则） +# -------------------------------------------------------------------------- + + +def alias_problem(alias: Any) -> str | None: + """别名能不能当内存库里的表名。能用返回 None。""" + if not isinstance(alias, str) or not alias: + return "别名不能为空" + if len(alias) > ALIAS_MAX: + return f"别名「{alias[:ALIAS_MAX]}…」太长，最多 {ALIAS_MAX} 个字符" + if not _ALIAS.match(alias): + return f"别名「{alias}」不是合法的表名：只能用英文字母、数字和下划线，且不能以数字开头" + if alias.lower() in _SQLITE_KEYWORDS: + return f"别名「{alias}」是 SQL 关键字，用作表名必须加引号，容易写错。请换一个，例如 sales、visits" + if alias.lower().startswith("sqlite_"): + return f"别名「{alias}」以 sqlite_ 开头，这是 SQLite 保留给内部表的名字，请换一个" + return None + + +def sql_problem(sql: Any) -> str | None: + """合并 SQL 有什么问题，没问题返回 None。{{ }} 运行时才有值，先按一个空值代进去检查写法。""" + if not isinstance(sql, str) or not sql.strip(): + return "还没有填写合并 SQL" + try: + _checked(_TEMPLATE.sub("NULL", sql)) + except MergeError as e: + return str(e) + return None + + +def _checked(sql: str) -> str: + """守卫（SQLite 读法）规范过的那一条语句。只认 SELECT / WITH；守卫的原话写给数据源的，换成合并查询自己的说法。""" + if not sql or not sql.strip(): + raise MergeError("还没有填写合并 SQL") + statements = guard.split_statements(sql, dialect="sqlite") + if len(statements) > 1: + raise MergeError(f"合并 SQL 只能写一条语句，这里有 {len(statements)} 条。请把它们合成一条 SELECT（可以用 WITH）") + try: + stmt = guard.check(sql, readonly=True, dialect="sqlite") + except SqlRejected as e: + # 只读拒绝的原话是「请在数据页另建一个可写的数据源」，合并查询用不上；结构变更（ATTACH、DROP）同理 + if guard.is_write(sql, dialect="sqlite"): + raise MergeError(f"{SELECT_ONLY}，不能写入、建表或连接别的库") from None + raise MergeError(f"合并 SQL 无法执行：{e}") from None + verb = guard.first_verb(stmt, dialect="sqlite") + if verb not in ("select", "with"): + raise MergeError(f"{SELECT_ONLY}，当前是 {verb.upper() or '无法识别的语句'}") + return stmt + + +# -------------------------------------------------------------------------- +# 执行 +# -------------------------------------------------------------------------- + + +def execute(inputs: list[MergeInput], sql: str, *, limits: QueryLimits | None = None) -> MergeOutcome: + """把输入落进内存库，执行合并 SQL。做不成抛 MergeError（消息给人看）。""" + limits = limits or QueryLimits() + if not inputs: + raise MergeError("还没有配置输入：至少选择一个上游的查询节点") + if len(inputs) > MAX_INPUTS: + raise MergeError(f"输入最多 {MAX_INPUTS} 个，这里有 {len(inputs)} 个。请先在源库里合并一部分") + _check_aliases(inputs) + tables = {inp.alias: _input_columns(inp) for inp in inputs} + stmt = _checked(sql) + + conn = open_memory_sqlite() + try: + rowids = {inp.alias: _load(conn, inp, tables[inp.alias]) for inp in inputs} + # 表落完就只读：授权回调之外的又一道（守卫、授权回调任何一处有漏洞，写也写不进去） + conn.execute("PRAGMA query_only = 1") + conn.set_authorizer(_authorize) + deadline = time.monotonic() + float(limits.timeout_seconds) + conn.set_progress_handler(lambda: 1 if time.monotonic() > deadline else 0, 1000) + columns, rows, truncated = _read(conn, stmt, limits) + plan = _lineage_plan(stmt, inputs, tables, rowids) + lineage = _lineage(conn, plan, columns, rows, inputs, tables) if plan is not None else None + except sqlite3.Error as e: + raise MergeError(_sqlite_reason(e, limits)) from None + finally: + conn.close() + + warnings = _warnings(stmt, inputs, tables, rows, truncated, limits) + return MergeOutcome( + columns=columns, rows=rows, truncated=truncated, sql=stmt, column_types=column_types(columns, rows), + warnings=warnings, lineage=lineage, inputs=[_summary(inp, tables[inp.alias]) for inp in inputs], + sources=_sources(inputs), mask_columns=_masks(inputs, columns, plan), + ) + + +def _check_aliases(inputs: list[MergeInput]) -> None: + seen: dict[str, str] = {} + for inp in inputs: + if problem := alias_problem(inp.alias): + raise MergeError(problem) + key = inp.alias.lower() + if key in seen: + raise MergeError(f"别名「{seen[key]}」和「{inp.alias}」只差大小写：SQLite 的表名不分大小写，会被当成同一张表。" + "请换一个别名") + seen[key] = inp.alias + + +def _who(inp: MergeInput) -> str: + return f"输入「{inp.alias}」" + (f"（{inp.label}）" if inp.label and inp.label != inp.alias else "") + + +def _input_columns(inp: MergeInput) -> list[str]: + snapshot = inp.snapshot + if not isinstance(snapshot, dict) or not isinstance(snapshot.get("columns"), list) \ + or not isinstance(snapshot.get("rows"), list): + raise MergeError(f"{_who(inp)}不是查询结果：快照里没有列和行") + if snapshot.get("truncated"): + raise MergeError( + f"{_who(inp)}的查询结果被截断，只拿到了前 {len(snapshot['rows'])} 行。拿不完整的结果按键合并，缺的行不会报错，" + "只会让合并结果悄悄少算。请在源库里先聚合到合并需要的粒度（例如按日期和门店 GROUP BY），" + "或者加条件缩小范围，让结果不超过查询上限") + columns = [str(c) for c in snapshot["columns"]] + if not columns: + raise MergeError(f"{_who(inp)}的查询结果没有列") + seen: dict[str, str] = {} + for name in columns: + key = name_key(name) + if key in seen: + raise MergeError(f"{_who(inp)}的结果里有重名的列「{name}」：SQLite 的列名不分大小写，同名的列放不进一张表。" + "请在源查询中用 AS 给它们起不同的名字") + seen[key] = name + return columns + + +def _quote(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def _kind(snapshot: dict[str, Any], column: str) -> str | None: + """这一列的类型：查询时记下的（column_types）优先；老快照没记的按值猜，猜不准不说。""" + types = snapshot.get("column_types") + if isinstance(types, dict) and isinstance(types.get(column), str): + return types[column] + columns = [str(c) for c in snapshot.get("columns") or []] + if column not in columns: + return None + at = columns.index(column) + kinds = {_value_kind(row[at]) for row in snapshot.get("rows") or [] + if isinstance(row, list) and at < len(row) and row[at] is not None} + return next(iter(kinds)) if len(kinds) == 1 and None not in kinds else None + + +def _value_kind(value: Any) -> str | None: + if isinstance(value, bool): + return "boolean" + if isinstance(value, (int, float)): + return "number" + return "text" if isinstance(value, str) else None + + +def _cell_in(value: Any) -> Any: + """快照里的值 → 写进内存库的值。快照是 JSON：布尔写成 0 / 1，对象和数组写成 JSON 文本。""" + if value is None or isinstance(value, (str, int, float)) and not isinstance(value, bool): + return value + if isinstance(value, bool): + return int(value) + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False, sort_keys=True) + return str(value) + + +def _load(conn: sqlite3.Connection, inp: MergeInput, columns: list[str]) -> str | None: + """建表、按快照的顺序写入行，返回这张表的 rowid 用哪个名字（第 i 行的 rowid 是 i + 1）。""" + snapshot = inp.snapshot + decl = ", ".join(f"{_quote(c)} {_AFFINITY.get(_kind(snapshot, c) or '', '')}".rstrip() for c in columns) + conn.execute(f"CREATE TABLE {_quote(inp.alias)} ({decl})") + taken = {c.lower() for c in columns} + rowid = next((n for n in _ROWID_NAMES if n not in taken), None) + width = len(columns) + rows: list[list[Any]] = [] + for i, row in enumerate(snapshot["rows"]): + if isinstance(row, dict): + row = [row.get(c) for c in columns] + if not isinstance(row, list) or len(row) != width: + raise MergeError(f"{_who(inp)}的快照第 {i} 行和列数对不上，快照可能已损坏") + rows.append([i + 1, *map(_cell_in, row)] if rowid else [_cell_in(v) for v in row]) + names = ", ".join([*([rowid] if rowid else []), *map(_quote, columns)]) + marks = ", ".join("?" * (width + (1 if rowid else 0))) + conn.executemany(f"INSERT INTO {_quote(inp.alias)} ({names}) VALUES ({marks})", rows) + return rowid + + +def _authorize(action: int, *_args: Any) -> int: + return sqlite3.SQLITE_OK if action in _ALLOWED_ACTIONS else sqlite3.SQLITE_DENY + + +def _read(conn: sqlite3.Connection, stmt: str, limits: QueryLimits) -> tuple[list[str], list[list[Any]], bool]: + """逐行读、边读边判上限：和查询层（data.engine.run_query）同一个口径，到了行数或字节上限就停，记 truncated。""" + cursor = conn.execute(stmt) + columns = [str(d[0]) for d in cursor.description or []] + if not columns: + raise MergeError("合并 SQL 没有返回任何列") + rows: list[list[Any]] = [] + size = 0 + truncated = False + for row in cursor: + values = [_jsonable(v) for v in row] + rows.append(values) + size += len(json.dumps(values, ensure_ascii=False, default=str)) + if len(rows) >= limits.max_rows or size >= limits.max_bytes: + truncated = True + break + return columns, rows, truncated + + +def _sqlite_reason(e: sqlite3.Error, limits: QueryLimits) -> str: + text = str(e) + if "interrupted" in text: + seconds = float(limits.timeout_seconds) + shown = f"{seconds:g}" + return f"合并超过 {shown} 秒被中断。请在源查询里先聚合，或在合并 SQL 中加条件缩小范围" + if "not authorized" in text or "readonly" in text: + return f"{SELECT_ONLY}，只能读取输入表" + # SQLite 的原话（no such column: x、ambiguous column name: 日期）正是改 SQL 要的线索，原样带上 + return f"合并 SQL 执行失败：{text}" + + +def _summary(inp: MergeInput, columns: list[str]) -> dict[str, Any]: + source = inp.snapshot.get("source") + return {"alias": inp.alias, "node_id": inp.node_id, "artifact": inp.artifact, "rows": len(inp.snapshot["rows"]), + "columns": columns, "source": str(source) if source else None} + + +def _sources(inputs: list[MergeInput]) -> list[str]: + out: list[str] = [] + for inp in inputs: + nested = inp.snapshot.get("sources") + names = nested if inp.snapshot.get("source") == MERGE_SOURCE and isinstance(nested, list) \ + else [inp.snapshot.get("source")] + out.extend(str(n) for n in names if n and str(n) not in out) + return out + + +# -------------------------------------------------------------------------- +# 逐格来历 +# -------------------------------------------------------------------------- + +#: 整条语句里出现就不追来历的词：子查询、集合运算、公用表表达式、分组、窗口、去重、NATURAL JOIN +_NO_LINEAGE = frozenset({"select", "union", "intersect", "except", "with", "recursive", "group", "having", "window", + "over", "distinct", "values", "natural", "filter"}) +#: 聚合函数（名字后面紧跟「(」）。没有 GROUP BY 的聚合把整张表压成一行，选取项里的裸列取的是哪一行说不准 +_AGGREGATES = frozenset({"count", "sum", "avg", "min", "max", "total", "group_concat", "string_agg", "median", + "json_group_array", "json_group_object", "jsonb_group_array", "jsonb_group_object"}) +#: 不加引号的字面量关键字：选取项写 NULL、CURRENT_DATE 不是列 +_LITERALS = frozenset({"null", "true", "false", "current_date", "current_time", "current_timestamp"}) +#: 来源表后面紧跟这些词时，它们不是省略了 AS 的表别名 +_AFTER_SOURCE = frozenset({"on", "using", "where", "order", "limit", "join", "left", "right", "full", "inner", "cross", + "outer", "indexed", "not", "natural", "group", "having", "window", "union", "intersect", + "except", "as"}) +_JOIN_WORDS = frozenset({"join", "left", "right", "full", "inner", "cross", "natural"}) +_TAIL = frozenset({"where", "order", "limit"}) + + +@dataclass(frozen=True) +class _Source: + #: 输入别名（内存库里的表名） + alias: str + #: SQL 里怎么称呼它：写了表别名就是表别名，否则是表名本身 + ref: str + + +@dataclass(frozen=True) +class _Plan: + sources: list[_Source] + #: 第 i 个结果列：(第几个来源, 输入里的列名)；表达式、说不准的列是 None + columns: list[tuple[int, str] | None] + #: 在原 SQL 的哪个位置补来历列（顶层 FROM 前面） + insert_at: int + stmt: str + rowids: list[str] + + +def _punct(lex: _Lex | None, text: str) -> bool: + return lex is not None and lex.kind == "punct" and lex.text == text + + +def _named(lex: _Lex | None) -> bool: + return lex is not None and lex.kind in ("word", "name") + + +def _find(names: list[str], wanted: str) -> str | None: + key = name_key(wanted) + return next((n for n in names if name_key(n) == key), None) + + +def _lineage_plan(stmt: str, inputs: list[MergeInput], tables: dict[str, list[str]], + rowids: dict[str, str | None]) -> _Plan | None: + """认出「单层 SELECT、选取项是输入表的列」这种写法；认不出返回 None（整份不记来历）。""" + toks = _lex(stmt) + if not toks or _kw(toks[0]) != "select": + return None + if any(_kw(t) in _NO_LINEAGE for t in toks[1:]): + return None + if any(_kw(t) in _AGGREGATES and _punct(nxt, "(") for t, nxt in zip(toks, toks[1:])): + return None + depth, from_at = 0, None + for i, t in enumerate(toks): + if _punct(t, "("): + depth += 1 + elif _punct(t, ")"): + depth -= 1 + elif depth == 0 and _kw(t) == "from": + from_at = i + break + if from_at is None: + return None + start = 2 if _kw(toks[1]) == "all" else 1 + parsed = _from_clause(toks, from_at + 1, tables) + if parsed is None: + return None + sources, using = parsed + columns: list[tuple[int, str] | None] = [] + for item in _split_items(toks[start:from_at]): + mapped = _item(item, sources, tables, using) + if mapped is None: + return None + columns.extend(mapped) + if any(rowids.get(s.alias) is None for s in sources): + return None + frm = toks[from_at] + return _Plan(sources=sources, columns=columns, insert_at=frm.end - len(frm.text), stmt=stmt, + rowids=[str(rowids[s.alias]) for s in sources]) + + +def _split_items(toks: list[_Lex]) -> list[list[_Lex]]: + items: list[list[_Lex]] = [[]] + depth = 0 + for t in toks: + if _punct(t, "("): + depth += 1 + elif _punct(t, ")"): + depth -= 1 + if depth == 0 and _punct(t, ","): + items.append([]) + else: + items[-1].append(t) + return items + + +def _from_clause(toks: list[_Lex], i: int, tables: dict[str, list[str]]) -> tuple[list[_Source], set[str]] | None: + """FROM 之后：来源表 [AS 别名]，用逗号或 JOIN 连接，JOIN 后面可跟 ON 条件或 USING (列…)；之后只许 + WHERE / ORDER BY / LIMIT。来源只能是输入表（不认子查询、表函数、带库名的表）。""" + sources: list[_Source] = [] + using: set[str] = set() + n = len(toks) + while True: + t = toks[i] if i < n else None + if not _named(t) or (t.kind == "word" and name_key(t.text) in _SQLITE_KEYWORDS): + return None + table = _find(list(tables), t.text) + if table is None or _punct(toks[i + 1] if i + 1 < n else None, ".") \ + or _punct(toks[i + 1] if i + 1 < n else None, "("): + return None + ref, i = t.text, i + 1 + nxt = toks[i] if i < n else None + if _kw(nxt) == "as": + if not _named(toks[i + 1] if i + 1 < n else None): + return None + ref, i = toks[i + 1].text, i + 2 + elif _named(nxt) and (nxt.kind == "name" or name_key(nxt.text) not in _AFTER_SOURCE): + ref, i = nxt.text, i + 1 + sources.append(_Source(alias=table, ref=ref)) + word = _kw(toks[i]) if i < n else None + if word == "on": + depth, i = 0, i + 1 + while i < n: + t = toks[i] + if _punct(t, "("): + depth += 1 + elif _punct(t, ")"): + depth -= 1 + elif depth == 0 and (_punct(t, ",") or _kw(t) in _JOIN_WORDS or _kw(t) in _TAIL): + break + i += 1 + elif word == "using": + if not _punct(toks[i + 1] if i + 1 < n else None, "("): + return None + i += 2 + while i < n and not _punct(toks[i], ")"): + if _named(toks[i]): + using.add(name_key(toks[i].text)) + elif not _punct(toks[i], ","): + return None + i += 1 + i += 1 + if i >= n: + return sources, using + t = toks[i] + if _punct(t, ","): + i += 1 + continue + word = _kw(t) + if word in _TAIL: + return sources, using + if word not in ("join", "left", "right", "full", "inner", "cross"): + return None + while i < n and _kw(toks[i]) in ("left", "right", "full", "inner", "cross", "outer"): + i += 1 + if i >= n or _kw(toks[i]) != "join": + return None + i += 1 + + +def _source_of(ref: str, sources: list[_Source]) -> int | None: + key = name_key(ref) + hits = [k for k, s in enumerate(sources) if name_key(s.ref) == key] + return hits[0] if len(hits) == 1 else None + + +def _item(item: list[_Lex], sources: list[_Source], tables: dict[str, list[str]], + using: set[str]) -> list[tuple[int, str] | None] | None: + """一个选取项展开成几个结果列的来历。返回 None 表示整份不追（* 遇上 USING：SQLite 只出一份合并掉的列）。""" + if len(item) == 1 and _punct(item[0], "*"): + if using: + return None + return [(k, c) for k, s in enumerate(sources) for c in tables[s.alias]] + if len(item) == 3 and _named(item[0]) and _punct(item[1], ".") and _punct(item[2], "*"): + k = _source_of(item[0].text, sources) + return None if k is None else [(k, c) for c in tables[sources[k].alias]] + body = item + if len(item) >= 3 and _kw(item[-2]) == "as" and _named(item[-1]): + body = item[:-2] + elif len(item) >= 2 and _named(item[-1]) and _named(item[-2]) \ + and not (item[-1].kind == "word" and name_key(item[-1].text) in _SQLITE_KEYWORDS): + body = item[:-1] + if len(body) == 1 and _named(body[0]): + word = body[0] + if word.kind == "word" and name_key(word.text) in _LITERALS: + return [None] + owners = [(k, col) for k, s in enumerate(sources) if (col := _find(tables[s.alias], word.text)) is not None] + if len(owners) != 1 or name_key(word.text) in using: + return [None] + return [owners[0]] + if len(body) == 3 and _named(body[0]) and _punct(body[1], ".") and _named(body[2]): + k = _source_of(body[0].text, sources) + col = _find(tables[sources[k].alias], body[2].text) if k is not None else None + return [(k, col) if k is not None and col is not None else None] + return [None] + + +def _hashable(row: list[Any]) -> str: + return json.dumps(row, ensure_ascii=False, sort_keys=True, default=str) + + +def _lineage(conn: sqlite3.Connection, plan: _Plan, columns: list[str], rows: list[list[Any]], + inputs: list[MergeInput], tables: dict[str, list[str]]) -> dict[str, Any] | None: + """补上 rowid 再执行一次，逐行逐值核对，得出每一格来自哪个输入的哪一行。核对不上返回 None。""" + width = len(columns) + if len(plan.columns) != width or not any(plan.columns): + return None + extra = ", ".join(f"{_quote(s.ref)}.{rowid} AS {_quote(_LINEAGE_COLUMN.format(k))}" + for k, (s, rowid) in enumerate(zip(plan.sources, plan.rowids))) + head = plan.stmt[:plan.insert_at].rstrip() + try: + augmented = _checked(f"{head}, {extra} {plan.stmt[plan.insert_at:]}") + cursor = conn.execute(augmented) + except (MergeError, sqlite3.Error): + return None + got: list[tuple[Any, ...]] = [] + for row in cursor: + if len(got) >= len(rows): + break + got.append(row) + if len(got) != len(rows) or any(len(g) != width + len(plan.sources) for g in got): + return None + if any([_jsonable(v) for v in g[:width]] != r for g, r in zip(got, rows)): + return None + origin: list[list[int | None]] = [[v - 1 if isinstance(v, int) and v >= 1 else None for v in g[width:]] + for g in got] + # 一模一样的两行结果：说不清哪一行来自哪一行输入 + counts = Counter(_hashable(r) for r in rows) + for i, r in enumerate(rows): + if counts[_hashable(r)] > 1: + origin[i] = [None] * len(plan.sources) + snaps = {inp.alias: inp.snapshot for inp in inputs} + mapped: list[dict[str, Any] | None] = [] + for j, target in enumerate(plan.columns): + if target is None: + mapped.append(None) + continue + k, column = target + snap = snaps[plan.sources[k].alias] + at = tables[plan.sources[k].alias].index(column) + ok = True + for i, row in enumerate(rows): + src = origin[i][k] + if src is None: + continue + if not (0 <= src < len(snap["rows"])) or not _same(row[j], _row(snap["rows"][src], tables[plan.sources[k].alias])[at]): + ok = False + break + mapped.append({"source": k, "column": column} if ok else None) + if not any(mapped): + return None + return {"sources": [{"input": s.alias, "ref": s.ref} for s in plan.sources], "columns": mapped, "rows": origin} + + +def _row(row: Any, columns: list[str]) -> list[Any]: + return [row.get(c) for c in columns] if isinstance(row, dict) else row + + +def _same(merged: Any, original: Any) -> bool: + if merged is None or original is None: + return merged is None and original is None + return same_value(merged, _cell_in(original)) + + +def trace_cell(snapshot: Any, row: Any, column: Any) -> Trace | None: + """合并结果的这一格来自哪个输入的哪一格；没有逐格来历返回 None。列按第一次出现的位置认（同 locate_cell）。""" + if not isinstance(snapshot, dict) or snapshot.get("source") != MERGE_SOURCE: + return None + lineage, columns = snapshot.get("lineage"), snapshot.get("columns") + if not isinstance(lineage, dict) or not isinstance(columns, list) or column not in columns: + return None + if isinstance(row, bool) or not isinstance(row, int): + return None + mapped, origin, sources = lineage.get("columns"), lineage.get("rows"), lineage.get("sources") + at = columns.index(column) + if not (isinstance(mapped, list) and isinstance(origin, list) and isinstance(sources, list) + and at < len(mapped) and 0 <= row < len(origin)): + return None + target = mapped[at] + if not isinstance(target, dict) or not isinstance(target.get("source"), int): + return None + k = target["source"] + line = origin[row] + if not isinstance(line, list) or not 0 <= k < len(line) or not 0 <= k < len(sources): + return None + src = line[k] + alias = (sources[k] or {}).get("input") if isinstance(sources[k], dict) else None + if isinstance(src, bool) or not isinstance(src, int) or not isinstance(alias, str): + return None + entry = next((e for e in snapshot.get("inputs") or [] if isinstance(e, dict) and e.get("alias") == alias), None) + if entry is None: + return None + return Trace(alias=alias, node_id=entry.get("node_id"), artifact=entry.get("artifact"), row=src, + column=str(target.get("column"))) + + +def _masks(inputs: list[MergeInput], columns: list[str], plan: _Plan | None) -> list[str]: + """合并结果要遮罩的列：各输入当时记下的遮罩列，加上来自遮罩列、换了名字的结果列（宁可多遮）。""" + hidden: dict[str, set[str]] = {} + names: list[str] = [] + for inp in inputs: + recorded = inp.snapshot.get("mask_columns") + cols = {str(c).lower() for c in recorded} if isinstance(recorded, list) else set() + hidden[inp.alias] = cols + names.extend(str(c) for c in recorded or [] if isinstance(c, str)) + if plan is not None and len(plan.columns) == len(columns): + for name, target in zip(columns, plan.columns): + if target is not None and target[1].lower() in hidden.get(plan.sources[target[0]].alias, set()): + names.append(name) + return sorted(dict.fromkeys(names)) + + +# -------------------------------------------------------------------------- +# 警告：合并键类型、行数放大、结果截断 +# -------------------------------------------------------------------------- + + +def _key_pairs(stmt: str, tables: dict[str, list[str]]) -> list[tuple[str, str, str, str]]: + """合并条件里的键：ON / WHERE 里「列 = 列」两边落在不同的输入表上，以及 USING (列)。只用来提示，认不全不要紧。""" + toks = _lex(stmt, strict=False) or [] + n = len(toks) + refs: dict[str, str] = {} + pairs: list[tuple[str, str, str, str]] = [] + placed: list[str] = [] # 当前 FROM 列表里已经出现过的输入表（USING 找左边那张） + i = 0 + while i < n: + word = _kw(toks[i]) + if word in ("from", "join") or (_punct(toks[i], ",") and placed and i > 0 and _named(toks[i - 1])): + if word == "from": + placed = [] + j = i + 1 + t = toks[j] if j < n else None + table = _find(list(tables), t.text) if _named(t) else None + if table is not None and not _punct(toks[j + 1] if j + 1 < n else None, "."): + ref, j = t.text, j + 1 + nxt = toks[j] if j < n else None + if _kw(nxt) == "as" and _named(toks[j + 1] if j + 1 < n else None): + ref, j = toks[j + 1].text, j + 2 + elif _named(nxt) and (nxt.kind == "name" or name_key(nxt.text) not in _AFTER_SOURCE): + ref, j = nxt.text, j + 1 + refs[name_key(ref)] = table + refs.setdefault(name_key(table), table) + if _kw(toks[j] if j < n else None) == "using" and _punct(toks[j + 1] if j + 1 < n else None, "("): + k = j + 2 + while k < n and not _punct(toks[k], ")"): + if _named(toks[k]): + col = _find(tables[table], toks[k].text) + left = next((p for p in reversed(placed) if _find(tables[p], toks[k].text)), None) + if col is not None and left is not None: + pairs.append((left, str(_find(tables[left], toks[k].text)), table, col)) + k += 1 + placed.append(table) + i = j + continue + i += 1 + + def resolve(at: int, step: int) -> tuple[tuple[str, str] | None, int]: + """从 at 开始（step=1 往右、-1 往左）读一个列引用：(表, 列)、它占了几个记号。""" + if step > 0: + seq = toks[at:at + 3] + else: + seq = toks[max(0, at - 2):at + 1] + if len(seq) == 3 and _named(seq[0]) and _punct(seq[1], ".") and _named(seq[2]): + table = refs.get(name_key(seq[0].text)) + col = _find(tables[table], seq[2].text) if table else None + return ((table, col) if table and col else None), 3 + one = toks[at] if 0 <= at < n else None + if _named(one): + owners = [(t, c) for t in tables if (c := _find(tables[t], one.text)) is not None] + return (owners[0] if len(owners) == 1 else None), 1 + return None, 1 + + for i, t in enumerate(toks): + if not _punct(t, "="): + continue + prev = toks[i - 1] if i > 0 else None + # 「==」的第二个等号、<=、>=、!= 都不是比较的开头（分词把每个标点切成一个记号） + if prev is None or (prev.kind == "punct" and prev.text in "=<>!"): + continue + right_at = i + 2 if _punct(toks[i + 1] if i + 1 < n else None, "=") else i + 1 + left, width = resolve(i - 1, -1) + before = toks[i - 1 - width] if i - 1 - width >= 0 else None + if before is not None and _punct(before, "."): + left = None # a.b.c 这种带库名的写法：不是输入表的列 + right, rwidth = resolve(right_at, 1) + after = toks[right_at + rwidth] if right_at + rwidth < n else None + if after is not None and (_punct(after, ".") or _punct(after, "(")): + right = None + if left and right and left[0] != right[0]: + pairs.append((left[0], left[1], right[0], right[1])) + seen: set[frozenset[tuple[str, str]]] = set() + out: list[tuple[str, str, str, str]] = [] + for a, ac, b, bc in pairs: + key = frozenset({(a, ac), (b, bc)}) + if key not in seen: + seen.add(key) + out.append((a, ac, b, bc)) + return out + + +def _example(snapshot: dict[str, Any], column: str) -> str | None: + columns = [str(c) for c in snapshot.get("columns") or []] + if column not in columns: + return None + at = columns.index(column) + for row in snapshot.get("rows") or []: + value = row[at] if isinstance(row, list) and at < len(row) else None + if value is not None: + return f"'{value}'" if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + return None + + +def _clash(a: str | None, b: str | None) -> bool: + """两边的类型放在一起比会不会出错：数值对文本类，日期对日期时间（'2026-05-01' 永远不等于 '2026-05-01T00:00:00'）。""" + if not a or not b or a == b: + return False + numeric = {"number", "boolean"} + if (a in numeric) != (b in numeric): + return True + return {a, b} == {"date", "datetime"} + + +def _warnings(stmt: str, inputs: list[MergeInput], tables: dict[str, list[str]], rows: list[list[Any]], + truncated: bool, limits: QueryLimits) -> list[dict[str, Any]]: + snaps = {inp.alias: inp.snapshot for inp in inputs} + pairs = _key_pairs(stmt, tables) + out: list[dict[str, Any]] = [] + for a, ac, b, bc in pairs: + ka, kb = _kind(snaps[a], ac), _kind(snaps[b], bc) + if not _clash(ka, kb): + continue + ea, eb = _example(snaps[a], ac), _example(snaps[b], bc) + left = f"{a}.{ac} 是{_KIND_LABEL.get(str(ka), ka)}" + (f"（例如 {ea}）" if ea else "") + right = f"{b}.{bc} 是{_KIND_LABEL.get(str(kb), kb)}" + (f"（例如 {eb}）" if eb else "") + if {ka, kb} == {"date", "datetime"}: + tail = ("日期和日期时间按文本比较永远不相等，这组键可能一行都匹配不上。请在源查询中统一格式，" + "或在合并 SQL 中用 date() 转换后再比较") + else: + tail = ("SQLite 比较时会做隐式转换：文本形式的编号（如 '001'、'01'）都会等于数值 1，也可能完全匹配不上。" + "请在源查询中统一类型，或在合并 SQL 中用 CAST 明确转换") + out.append({"code": "key_type_mismatch", "keys": [f"{a}.{ac}", f"{b}.{bc}"], + "message": f"合并键类型不一致：{left}，{right}。{tail}"}) + largest = max(inputs, key=lambda inp: len(inp.snapshot["rows"])) + biggest = len(largest.snapshot["rows"]) + if len(rows) > biggest: + keys: dict[str, list[str]] = {} + for a, ac, b, bc in pairs: + for alias, col in ((a, ac), (b, bc)): + if col not in keys.setdefault(alias, []): + keys[alias].append(col) + dups = [_duplicates(alias, snaps[alias], cols) for alias, cols in keys.items()] + detail = "；".join(d for d in dups if d) + out.append({"code": "rows_grew", "message": ( + f"合并结果有 {len(rows)} 行，多于行数最多的输入「{largest.alias}」（{biggest} 行）：合并键可能不唯一，" + "同一行被重复匹配。" + (f"{detail}。" if detail else "") + + "请检查合并条件是否覆盖了全部键（例如同时按日期和门店），或先在源库里聚合到相同粒度")}) + if truncated: + out.append({"code": "result_truncated", "message": ( + f"合并结果超过上限（{limits.max_rows} 行或约 {max(1, limits.max_bytes // 1_000_000)} MB），只保留了前 " + f"{len(rows)} 行，下游拿到的不是完整结果。请在合并 SQL 中聚合或加条件缩小范围")}) + return out + + +def _duplicates(alias: str, snapshot: dict[str, Any], cols: list[str]) -> str | None: + """这个输入按合并键有没有重复：有的话举第一组，说一共几组。""" + columns = [str(c) for c in snapshot.get("columns") or []] + if not cols or any(c not in columns for c in cols): + return None + at = [columns.index(c) for c in cols] + counts = Counter(tuple(_hashable([row[j]]) for j in at) for row in snapshot.get("rows") or [] + if isinstance(row, list)) + repeated = [(key, n) for key, n in counts.items() if n > 1] + if not repeated: + return None + key, n = repeated[0] + shown = " / ".join(str(json.loads(v)[0]) for v in key) + more = f"等 {len(repeated)} 组" if len(repeated) > 1 else "" + return f"「{alias}」中按（{'、'.join(cols)}）有重复的键，例如 {shown} 出现 {n} 次{more}" diff --git a/backend/app/engine/nodes/merge.py b/backend/app/engine/nodes/merge.py new file mode 100644 index 00000000..fc668689 --- /dev/null +++ b/backend/app/engine/nodes/merge.py @@ -0,0 +1,132 @@ +"""合并查询节点：读上游的查询快照，在库外按键合并，结果存成和数据源查询同形的查询快照。 + +**读的是工件库里那份完整结果。** 上游 tool 节点交回的 JSON 里带着查询快照的工件 id，这里按 id 取回快照（取回时复验 +哈希），不读交给模型的那份文字：Agent 的消息历史里结果可能被截短，拿截短的文字合并会悄悄少行。上游是另一个合并 +节点时同理，取它的结果快照。 + +**结果和数据源查询同形。** 快照有 columns、rows、truncated、sql、source（记为「合并查询」），台账里记一条 kind 为 +query 的条目：报告照样按先后编成 Qn，[[v:Qn.rK.列]]、[[table:Qn …]] 和口径卡的 cell(nodes.<本节点>, 行, '列') 都按 +原来的规则取值、核对。快照另记 inputs（别名、节点、各自的快照）和合并 SQL，证据面板据此展示合并的来源；逐格来历 +记在 lineage 里（见 engine/merge_query.py），下钻据此指回输入的那一格。 + +**重放结果一致。** 快照里没有耗时这类每次不同的字段：节点重放、继续运行时同样的输入得到同一个工件 id、同一条 +台账，报告里已经写下的出处不会因此变成另一份快照。 +""" +from __future__ import annotations + +import asyncio +import json +import time +from typing import Any + +from app.core import artifact_store +from app.core.events import EventType +from app.data.guard import QueryLimits +from app.engine.context import NodeContext, NodeError +from app.engine.evidence import ledger_enabled, next_exec +from app.engine.merge_query import MERGE_SOURCE, MergeError, MergeInput, execute +from app.engine.state import GraphState + +#: 合并警告进运行事件时的 code（log 事件）。运行详情、运行概况按它认出这是合并查询的警告 +LOG_CODES = {"key_type_mismatch": "merge_key_type", "rows_grew": "merge_rows_grew", + "result_truncated": "merge_truncated"} +#: merge.end 事件里带几行预览：运行详情只是让人看一眼合并出了什么，完整结果在查询快照里 +PREVIEW_ROWS = 5 +#: 上游查询没查成时，tool 节点交回的原话的开头（engine/nodes/tools.py 的 QUERY_FAILED 同一组） +_FAILED_PREFIXES = ("查询失败", "SQL 被拒绝：") + + +def _input_payload(value: Any) -> dict[str, Any] | None: + """上游节点的产出 → 带工件 id 的查询结果（tool 节点交回的是 JSON 文本，合并节点交回的是对象）。""" + data = value + if isinstance(data, str): + try: + data = json.loads(data) + except ValueError: + return None + return data if isinstance(data, dict) else None + + +async def _load_input(state: GraphState, ctx: NodeContext, alias: str, node_id: Any) -> MergeInput: + nodes = ctx.run.spec.node_map() + target = nodes.get(node_id) if isinstance(node_id, str) else None + if target is None: + raise NodeError(ctx.node.id, f"输入「{alias}」选择的节点「{node_id}」不存在，请在节点中重新选择") + who = f"输入「{alias}」（{target.title}）" + value = (state.get("nodes") or {}).get(target.id) + if value is None: + raise NodeError(ctx.node.id, f"{who}还没有运行结果：请确认它连在合并查询之前") + if isinstance(value, dict) and value.get("skipped"): + raise NodeError(ctx.node.id, f"{who}满足跳过条件被跳过了，没有查询结果可以合并") + if isinstance(value, dict) and value.get("failed"): + raise NodeError(ctx.node.id, f"{who}执行失败，没有查询结果可以合并：{value.get('error') or '原因未知'}") + if isinstance(value, str) and value.startswith(_FAILED_PREFIXES): + raise NodeError(ctx.node.id, f"{who}的查询没有成功，无法合并：{value[:300]}") + payload = _input_payload(value) + artifact = payload.get("artifact") if payload else None + if not isinstance(artifact, str) or not artifact: + raise NodeError(ctx.node.id, f"{who}的结果不是存成快照的查询结果。合并查询只能合并数据库查询（调用数据库查询工具的" + "「调用工具」节点）或另一个「合并查询」节点的结果") + try: + snapshot = await asyncio.to_thread(artifact_store.load, artifact) + except ValueError: + raise NodeError(ctx.node.id, f"{who}的查询快照与哈希不一致，疑似被修改，不能合并") from None + if snapshot is None: + raise NodeError(ctx.node.id, f"{who}的查询快照已不存在，无法合并。请重新运行它") + return MergeInput(alias=alias, node_id=target.id, artifact=artifact, snapshot=snapshot, label=target.title) + + +async def run_merge(state: GraphState, ctx: NodeContext) -> dict[str, Any]: + inputs = ctx.config.get("inputs") + if not isinstance(inputs, dict) or not inputs: + raise NodeError(ctx.node.id, "「合并查询」还没有配置输入：至少选择一个上游的查询节点，并给它起一个别名") + # SQL 里允许 {{ }}（例如按运行输入筛日期），和 tool 节点的 SQL 一样先渲染；快照里存的是渲染之后、守卫规范过的那条 + sql = ctx.render_str(str(ctx.config.get("sql") or ""), state) + loaded = [await _load_input(state, ctx, str(alias), node_id) for alias, node_id in inputs.items()] + + started = time.perf_counter() + try: + # 内存 SQLite 是同步的：放进线程，大一点的合并不卡住事件循环（别的节点的事件照常流出去） + outcome = await asyncio.to_thread(execute, loaded, sql, limits=QueryLimits()) + except MergeError as e: + raise NodeError(ctx.node.id, str(e)) from None + elapsed = int((time.perf_counter() - started) * 1000) + + snapshot = outcome.snapshot() + try: + artifact = await artifact_store.put_json(snapshot, kind="query_snapshot", run_id=ctx.run.run_id, + node_id=ctx.node.id) + except Exception as e: # noqa: BLE001 - 存不下就没有出处可言，不能带着一份引用不了的结果往下走 + raise NodeError(ctx.node.id, "合并结果没能存成查询快照（工件库写入失败），下游无法引用它。" + "请稍后点「继续运行」重试", retryable=True) from e + + labels = {inp.alias: inp.label for inp in loaded} + inputs_view = [{**{k: v for k, v in item.items() if k != "columns"}, "label": labels.get(item["alias"]) or None} + for item in outcome.inputs] + ctx.emit(EventType.MERGE_END, inputs=inputs_view, sql=outcome.sql, rows=len(outcome.rows), + columns=outcome.columns, truncated=outcome.truncated, warnings=outcome.warnings, + preview_rows=outcome.rows[:PREVIEW_ROWS], duration_ms=elapsed, query_artifact=artifact, + lineage=outcome.lineage is not None) + for warning in outcome.warnings: + ctx.emit(EventType.LOG, level="warn", message=warning["message"], + code=LOG_CODES.get(warning["code"], "merge_warning")) + + output: dict[str, Any] = { + "artifact": artifact, "columns": outcome.columns, "rows": outcome.rows, "row_count": len(outcome.rows), + "truncated": outcome.truncated, "sql": outcome.sql, "source": MERGE_SOURCE, "inputs": inputs_view, + "warnings": outcome.warnings, + } + if outcome.column_types: + output["column_types"] = outcome.column_types + updates: dict[str, Any] = {"nodes": {ctx.node.id: output}} + if ledger_enabled(ctx.run): + # 证据台账：和数据源查询同形的 query 条目，报告据此编 Qn；没有 via（外面不包工具快照） + updates["evidence"] = [{ + "kind": "query", "node_id": ctx.node.id, "exec": next_exec(state, ctx.node.id), "artifact": artifact, + "source": MERGE_SOURCE, "columns": outcome.columns, "rows": len(outcome.rows), + "truncated": outcome.truncated, + }] + var_name = str(ctx.config.get("assign_to") or "").strip() + if var_name: + updates["vars"] = {var_name: output} + return updates diff --git a/backend/app/engine/nodes/metrics.py b/backend/app/engine/nodes/metrics.py index d3826b12..8d5909bb 100644 --- a/backend/app/engine/nodes/metrics.py +++ b/backend/app/engine/nodes/metrics.py @@ -388,7 +388,7 @@ _REF_HEAD = re.compile(r"^(vars|nodes|input)\.([A-Za-z_一-鿿][\w\-一-鿿]*)(. _VIA = {"input": "input", "code": "code", "agent": "agent", "llm": "llm", "tool": "tool", "transform": "transform", "metrics": "metric", "retrieve": "retrieval", "loop": "loop", "human": "human", "validate": "validate", "supervisor": "team", "subgraph": "subgraph", - "memory": "memory"} + "memory": "memory", "merge": "merge"} def _input_of(path: str, value: Any, state: GraphState, ctx: NodeContext, diff --git a/backend/app/engine/schema.py b/backend/app/engine/schema.py index a0be2442..e756edb4 100644 --- a/backend/app/engine/schema.py +++ b/backend/app/engine/schema.py @@ -23,6 +23,7 @@ class NodeType(StrEnum): TOOL = "tool" # 直接调用一个工具 CODE = "code" # 沙箱里跑代码 + MERGE = "merge" # 合并查询：几次查询的完整结果在库外按键合并（engine/merge_query.py） BRANCH = "branch" # 条件分支 LOOP = "loop" # 循环 / 迭代 @@ -416,6 +417,70 @@ def _check_report_sources(node: GraphNode, spec: GraphSpec, result: ValidationRe node_id=node.id, field="metrics_from", code=code) +def _produces_query(node: GraphNode) -> bool: + """这个节点的产出是一份存成快照的查询结果：调用数据库查询工具的「调用工具」节点，或另一个合并查询。""" + if node.type == NodeType.MERGE: + return True + return node.type == NodeType.TOOL and str(node.config.get("tool") or "").startswith("db_query__") + + +def _check_merge(node: GraphNode, spec: GraphSpec, result: ValidationResult) -> None: + """合并查询的输入和合并 SQL。规则和执行时（engine/merge_query.py）同一套，画图时就说破，不等运行到这一步才失败。 + + 输入只认能稳定取到一份查询快照的节点。Agent 查过的库不行：一次循环里可能查好几次，取哪一次说不准， + 拿错一次合并出来的数照样像模像样——所以报 error，并说清楚把那条查询单独放进「调用工具」节点。 + """ + from app.engine.merge_query import MAX_INPUTS, alias_problem, sql_problem + + cfg = node.config + inputs = cfg.get("inputs") + if not isinstance(inputs, dict) or not inputs: + result.add("「合并查询」还没有配置输入：至少选择一个上游的查询节点，并给它起一个别名（合并 SQL 里的表名）", + node_id=node.id, field="inputs", code="merge.inputs_missing") + else: + if len(inputs) > MAX_INPUTS: + result.add(f"「合并查询」最多 {MAX_INPUTS} 个输入，这里有 {len(inputs)} 个。请先在源库里合并一部分", + node_id=node.id, field="inputs", code="merge.inputs_missing") + nodes = spec.node_map() + upstream = _ancestors(spec, node.id) + seen: dict[str, str] = {} + for alias, source in inputs.items(): + field = f"inputs.{alias}" + if problem := alias_problem(alias): + result.add(f"「合并查询」的{problem}", node_id=node.id, field=field, code="merge.alias_invalid") + elif alias.lower() in seen: + result.add(f"「合并查询」的别名「{seen[alias.lower()]}」和「{alias}」只差大小写：SQLite 的表名不分大小写，" + "会被当成同一张表。请换一个别名", node_id=node.id, field=field, code="merge.alias_invalid") + else: + seen[alias.lower()] = alias + target = nodes.get(source) if isinstance(source, str) else None + who = f"「合并查询」的输入「{alias}」" + if not isinstance(source, str) or not source: + result.add(f"{who}还没有选择节点", node_id=node.id, field=field, code="merge.input_invalid") + elif target is None: + result.add(f"{who}选择了「{source}」，但工作流中不存在这个节点", node_id=node.id, field=field, + code="merge.input_invalid") + elif target.type == NodeType.AGENT: + result.add(f"{who}选择了「{target.title}」（Agent）：Agent 可能查询多次，无法确定合并哪一次的结果。" + "请把要合并的那条查询单独放进「调用工具」节点（选择数据库查询工具），再把它选作输入", + node_id=node.id, field=field, code="merge.input_invalid") + elif target.type == NodeType.TOOL and not _produces_query(target): + result.add(f"{who}选择的「{target.title}」调用的不是数据库查询工具：合并查询只能合并数据库查询的结果", + node_id=node.id, field=field, code="merge.input_invalid") + elif not _produces_query(target): + result.add(f"{who}选择的「{target.title}」是「{type_label(target.type)}」节点，不产出查询结果。" + "输入只能是调用数据库查询工具的「调用工具」节点，或另一个「合并查询」节点", + node_id=node.id, field=field, code="merge.input_invalid") + elif source not in upstream: + result.add(f"{who}选择的「{target.title}」不在合并查询的上游：合并时它还没有运行。请把它连到合并查询之前", + node_id=node.id, field=field, code="merge.input_invalid") + sql = cfg.get("sql") + if not isinstance(sql, str) or not sql.strip(): + result.add("「合并查询」还没有填写合并 SQL", node_id=node.id, field="sql", code="merge.sql_invalid") + elif problem := sql_problem(sql): + result.add(problem, node_id=node.id, field="sql", code="merge.sql_invalid") + + #: 报告撰写节点的结论句策略（claims）。off：结论句不参与判档（默认，保持以前的行为）； #: require_citation：没挂引用的结论句计入缺口、按出具档位降档。这两档是确定性的（发布前自动修复 #: 给人选的也只有这两档） @@ -604,6 +669,8 @@ def validate_graph(spec: GraphSpec) -> ValidationResult: elif node.type == NodeType.REPORT: _check_report_sources(node, spec, result) _check_report_claims(node, spec, result) + elif node.type == NodeType.MERGE: + _check_merge(node, spec, result) elif node.type == NodeType.OUTPUT: contract = cfg.get("contract") if contract and not isinstance(contract, dict): diff --git a/backend/app/engine/upgrade.py b/backend/app/engine/upgrade.py index 022e39a5..2bb86478 100644 --- a/backend/app/engine/upgrade.py +++ b/backend/app/engine/upgrade.py @@ -53,7 +53,7 @@ _LLM_ONLY = ("prompt", "system", "skills", "use_history", "output_schema") _MODEL_KEYS = ("provider", "model") #: 报告的指令里照搬了这几类节点的文字，就要提醒：报告撰写自己会收集它们的证据 _EVIDENCE_TYPES = (NodeType.METRICS, NodeType.TOOL, NodeType.AGENT, NodeType.RETRIEVE, NodeType.CODE, - NodeType.SUBGRAPH) + NodeType.SUBGRAPH, NodeType.MERGE) #: 写整段文字的节点：出具契约的 narrative 拼了其中几个的文字，就说不准该换哪一个 _WRITER_TYPES = (NodeType.LLM, NodeType.AGENT) _QUESTION = re.compile(r"(? bool: """这个节点查库：报告撰写能把它查到的结果编成 Q1、Q2… 引用。只认数据源的查询工具，别的工具的 - 返回进不了目录。""" + 返回进不了目录。合并查询的结果和数据源查询同形，同样编号。""" + if node.type == NodeType.MERGE: + return True if node.type == NodeType.TOOL: return str(node.config.get("tool") or "").startswith("db_query__") if node.type == NodeType.AGENT: diff --git a/backend/app/engine/variables.py b/backend/app/engine/variables.py index f023f4d3..8f95a20d 100644 --- a/backend/app/engine/variables.py +++ b/backend/app/engine/variables.py @@ -340,6 +340,15 @@ def analyze(spec: GraphSpec) -> VariableReport: node_id=node.id, node_label=label, field=field, expr=m.group(0), )) + # 合并查询的输入按节点 id 选，不写模板：不认这一条的话，变量表里看不出上游那条查询被谁用了。 + # 选错的节点（不存在、不在上游、不是查询）由 validate_graph 报，这里只记引用 + if node.type == "merge" and isinstance(cfg.get("inputs"), dict): + for alias, source in cfg["inputs"].items(): + target = index.get(f"nodes.{source}") if isinstance(source, str) else None + if target is not None: + target.refs.append(VarRef(node_id=node.id, node_label=label, field=f"inputs.{alias}", + expr=f"nodes.{source}")) + for field, text in _iter_strings(cfg): for m in _TEMPLATE_RE.finditer(text): expr = m.group(1).strip() diff --git a/backend/tests/fixtures/merge/members.sql b/backend/tests/fixtures/merge/members.sql new file mode 100644 index 00000000..e0ac8130 --- /dev/null +++ b/backend/tests/fixtures/merge/members.sql @@ -0,0 +1,28 @@ +-- 会员库（虚构）：会员到店记录，和门店库是两个独立的库。门店编号同样是文本，但列名不同（store_code）。 +-- 合并查询的端到端测试按「日期 + 门店」聚合到店人数，再和门店库的订单按键合并。数据只为测试编造。 +CREATE TABLE visits ( + visit_id INTEGER PRIMARY KEY, + member_id INTEGER NOT NULL, + store_code TEXT NOT NULL, + visit_date TEXT NOT NULL +); +INSERT INTO visits VALUES + (1, 101, 'S01', '2026-05-01'), + (2, 102, 'S01', '2026-05-01'), + (3, 103, 'S01', '2026-05-01'), + (4, 104, 'S01', '2026-05-01'), + (5, 105, 'S01', '2026-05-01'), + (6, 106, 'S01', '2026-05-01'), + (7, 201, 'S02', '2026-05-01'), + (8, 202, 'S02', '2026-05-01'), + (9, 203, 'S02', '2026-05-01'), + (10, 204, 'S02', '2026-05-01'), + (11, 101, 'S01', '2026-05-02'), + (12, 107, 'S01', '2026-05-02'), + (13, 108, 'S01', '2026-05-02'), + (14, 109, 'S01', '2026-05-02'), + (15, 110, 'S01', '2026-05-02'), + (16, 201, 'S02', '2026-05-02'), + (17, 205, 'S02', '2026-05-02'), + (18, 206, 'S02', '2026-05-02'), + (19, 207, 'S02', '2026-05-02'); diff --git a/backend/tests/fixtures/merge/stores.sql b/backend/tests/fixtures/merge/stores.sql new file mode 100644 index 00000000..ad097469 --- /dev/null +++ b/backend/tests/fixtures/merge/stores.sql @@ -0,0 +1,22 @@ +-- 门店库（虚构）：门店和销售订单。合并查询的端到端测试按「日期 + 门店」聚合订单数和销售额。 +-- 门店编号是文本（'S01'），金额是实数。数据只为测试编造，和任何真实门店无关。 +CREATE TABLE stores ( + store_id TEXT PRIMARY KEY, + name TEXT NOT NULL +); +CREATE TABLE orders ( + order_id INTEGER PRIMARY KEY, + store_id TEXT NOT NULL REFERENCES stores (store_id), + order_date TEXT NOT NULL, + amount REAL NOT NULL +); +INSERT INTO stores VALUES ('S01', '湖滨店'), ('S02', '东站店'); +INSERT INTO orders VALUES + (1, 'S01', '2026-05-01', 120.0), + (2, 'S01', '2026-05-01', 80.0), + (3, 'S01', '2026-05-01', 100.0), + (4, 'S02', '2026-05-01', 60.0), + (5, 'S02', '2026-05-01', 90.0), + (6, 'S01', '2026-05-02', 200.0), + (7, 'S01', '2026-05-02', 50.0), + (8, 'S02', '2026-05-02', 75.0); diff --git a/backend/tests/test_merge_engine.py b/backend/tests/test_merge_engine.py new file mode 100644 index 00000000..55bda2be --- /dev/null +++ b/backend/tests/test_merge_engine.py @@ -0,0 +1,385 @@ +"""合并查询的库外合并（engine/merge_query.py）：几次查询的完整快照落进内存 SQLite，再按一条 SELECT / WITH 合并。 + +纯函数，不连库、不读工件：快照在测试里现造。端到端（真运行、证据面板、推断来源）见 test_merge_node.py。 + +场景是虚构的门店经营：门店库按「日期 + 门店」聚合的销售，会员库按同样粒度聚合的到店人数。 +""" +from __future__ import annotations + +import pytest + +from app.data.guard import QueryLimits +from app.engine.merge_query import ( + MERGE_SOURCE, + MergeError, + MergeInput, + alias_problem, + execute, + sql_problem, + trace_cell, +) +from app.engine.expressions import same_value + +SALES_COLS = ["日期", "门店", "订单数", "销售额"] +SALES = [ + ["2026-05-01", "S01", 40, "3200.50"], + ["2026-05-01", "S02", 25, "1875.00"], + ["2026-05-02", "S01", 38, "3010.00"], + ["2026-05-02", "S02", 30, "2400.00"], +] +VISITS_COLS = ["日期", "门店", "到店人数"] +VISITS = [ + ["2026-05-01", "S01", 160], + ["2026-05-01", "S02", 90], + ["2026-05-02", "S02", 100], + ["2026-05-02", "S01", 150], +] +JOIN_SQL = ("SELECT s.日期, s.门店, s.订单数, s.销售额, v.到店人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 " + "ORDER BY s.日期, s.门店") + + +def snap(columns, rows, *, types=None, truncated=False, source="stores", **extra): + return {"columns": list(columns), "rows": [list(r) for r in rows], "row_count": len(rows), "truncated": truncated, + "sql": "SELECT …", "source": source, **({"column_types": types} if types is not None else {}), **extra} + + +def sales(**kw): + return MergeInput(alias="s", node_id="q_sales", artifact="a" * 64, + snapshot=snap(SALES_COLS, SALES, types={"日期": "text", "门店": "text", "订单数": "number", + "销售额": "number"}, **kw)) + + +def visits(rows=VISITS, *, types=None, **kw): + return MergeInput(alias="v", node_id="q_visits", artifact="b" * 64, + snapshot=snap(VISITS_COLS, rows, types=types or {"日期": "text", "门店": "text", "到店人数": "number"}, + source="members", **kw)) + + +# -------------------------------------------------------------------------- +# 合并本身 +# -------------------------------------------------------------------------- + + +def test_join_on_date_and_store(): + out = execute([sales(), visits()], JOIN_SQL) + assert out.columns == ["日期", "门店", "订单数", "销售额", "到店人数"] + assert out.rows == [ + ["2026-05-01", "S01", 40, 3200.5, 160], + ["2026-05-01", "S02", 25, 1875, 90], + ["2026-05-02", "S01", 38, 3010, 150], + ["2026-05-02", "S02", 30, 2400, 100], + ] + # 数值列按数存：数据源把 DECIMAL 落成的文本（"3200.50"）合并之后能直接参与运算 + assert out.column_types == {"日期": "text", "门店": "text", "订单数": "number", "销售额": "number", + "到店人数": "number"} + assert out.truncated is False and out.warnings == [] + assert out.sql == JOIN_SQL + + +def test_with_is_allowed_and_text_keeps_leading_zeros(): + a = MergeInput(alias="a", node_id="n1", artifact="c" * 64, + snapshot=snap(["会员号", "积分"], [["007", 10], ["070", 20]], types={"会员号": "text", "积分": "number"})) + out = execute([a], "WITH t AS (SELECT 会员号, 积分 FROM a) SELECT 会员号, 积分 * 2 AS 双倍 FROM t ORDER BY 会员号") + assert out.rows == [["007", 20], ["070", 40]] + + +def test_column_names_are_checked_not_treated_as_strings(): + """内存库同样关掉「双引号当字符串」：写错的列名要报错，不能静默得出一列常量。""" + with pytest.raises(MergeError, match="no such column"): + execute([sales()], 'SELECT "销售颔" FROM s') + + +@pytest.mark.parametrize("sql", [ + "DELETE FROM s", + "UPDATE s SET 订单数 = 0", + "INSERT INTO s VALUES (1, 2, 3, 4)", + "DROP TABLE s", + "PRAGMA table_info(s)", + "ATTACH DATABASE '/tmp/x.db' AS x", + "WITH k AS (SELECT 1) DELETE FROM s", + "EXPLAIN SELECT * FROM s", +]) +def test_only_select_and_with(sql): + assert sql_problem(sql) is not None + with pytest.raises(MergeError): + execute([sales()], sql) + + +def test_one_statement_only(): + problem = sql_problem("SELECT * FROM s; SELECT * FROM v") + assert problem is not None and "一条" in problem + with pytest.raises(MergeError): + execute([sales(), visits()], "SELECT * FROM s; SELECT * FROM v") + + +def test_empty_sql(): + assert "合并 SQL" in sql_problem(" ") + + +def test_static_check_tolerates_templates(): + assert sql_problem("SELECT * FROM s WHERE 日期 >= {{ input.start | json }}") is None + + +@pytest.mark.parametrize("alias, ok", [ + ("s", True), ("sales_2026", True), ("_t", True), + ("", False), ("1s", False), ("a-b", False), ("a b", False), ("订单", False), + ("select", False), ("ORDER", False), ("join", False), ("sqlite_stat", False), ("x" * 40, False), +]) +def test_alias_must_be_a_plain_table_name(alias, ok): + assert (alias_problem(alias) is None) is ok + + +def test_aliases_differing_only_in_case_collide(): + a = MergeInput(alias="t", node_id="n1", artifact="c" * 64, snapshot=snap(["x"], [[1]])) + b = MergeInput(alias="T", node_id="n2", artifact="d" * 64, snapshot=snap(["x"], [[2]])) + with pytest.raises(MergeError, match="别名"): + execute([a, b], "SELECT * FROM t") + + +def test_duplicate_column_names_in_an_input_are_refused_with_a_fix(): + a = MergeInput(alias="t", node_id="n1", artifact="c" * 64, snapshot=snap(["金额", "金额"], [[1, 2]])) + with pytest.raises(MergeError, match="重名"): + execute([a], "SELECT * FROM t") + + +# -------------------------------------------------------------------------- +# 三道防护 +# -------------------------------------------------------------------------- + + +def test_a_truncated_input_is_refused(): + """被截断的输入只有前若干行：拿它按键合并，缺的行不会报错，只会悄悄少算。""" + with pytest.raises(MergeError) as e: + execute([sales(), visits(truncated=True)], JOIN_SQL) + message = str(e.value) + assert "「v」" in message and "截断" in message + assert "聚合" in message and "缩小范围" in message + + +def test_key_type_mismatch_is_warned(): + """门店编号一边是文本 '001'，一边是数 1：SQLite 比较时会隐式转换，'001'、'01' 都等于 1。""" + stores = MergeInput(alias="s", node_id="q_sales", artifact="a" * 64, + snapshot=snap(["门店", "销售额"], [["001", 100], ["002", 200]], + types={"门店": "text", "销售额": "number"})) + traffic = MergeInput(alias="v", node_id="q_visits", artifact="b" * 64, + snapshot=snap(["门店", "到店人数"], [[1, 10], [2, 20]], types={"门店": "number", "到店人数": "number"}, + source="members")) + out = execute([stores, traffic], "SELECT s.门店, s.销售额, v.到店人数 FROM s JOIN v ON s.门店 = v.门店") + [warning] = out.warnings + assert warning["code"] == "key_type_mismatch" + assert "s.门店" in warning["message"] and "v.门店" in warning["message"] + assert "'001'" in warning["message"] and "CAST" in warning["message"] + + +def test_key_type_mismatch_is_found_through_using(): + stores = MergeInput(alias="s", node_id="n1", artifact="a" * 64, + snapshot=snap(["门店", "销售额"], [["1", 100]], types={"门店": "text", "销售额": "number"})) + traffic = MergeInput(alias="v", node_id="n2", artifact="b" * 64, + snapshot=snap(["门店", "到店人数"], [[1, 10]], types={"门店": "number", "到店人数": "number"})) + out = execute([stores, traffic], "SELECT 门店, 销售额, 到店人数 FROM s JOIN v USING (门店)") + assert [w["code"] for w in out.warnings] == ["key_type_mismatch"] + + +def test_matching_key_types_raise_no_warning(): + out = execute([sales(), visits()], JOIN_SQL) + assert out.warnings == [] + + +def test_more_rows_than_the_largest_input_is_warned(): + """两边都有一天同一门店出现两行（没聚合到同一粒度）：多对多匹配，合并后行数被放大。""" + stores = sales() + stores.snapshot["rows"].append(["2026-05-01", "S01", 2, "100.00"]) + doubled = [*VISITS, ["2026-05-01", "S01", 5]] + out = execute([stores, visits(doubled)], JOIN_SQL) + assert len(out.rows) == 7 + grew = [w for w in out.warnings if w["code"] == "rows_grew"] + assert len(grew) == 1 + message = grew[0]["message"] + assert "7 行" in message and "5 行" in message and "不唯一" in message + # 合并键认得出来时，说出是哪个输入、哪组键重复 + assert "「v」" in message and "2026-05-01" in message and "S01" in message + + +def test_one_to_many_is_not_warned_as_growth(): + """一边的键唯一时，结果行数不会超过另一边：不报。""" + doubled = [*VISITS, ["2026-05-01", "S01", 5]] + out = execute([sales(), visits(doubled)], JOIN_SQL) + assert len(out.rows) == 5 and out.warnings == [] + + +def test_result_over_the_row_limit_is_truncated_and_warned(): + out = execute([sales(), visits()], "SELECT * FROM s CROSS JOIN v", limits=QueryLimits(max_rows=5)) + assert out.truncated is True and len(out.rows) == 5 + assert "result_truncated" in [w["code"] for w in out.warnings] + + +def test_byte_limit_matches_the_query_layer(): + out = execute([sales(), visits()], "SELECT * FROM s CROSS JOIN v", limits=QueryLimits(max_bytes=100)) + assert out.truncated is True and 0 < len(out.rows) < 16 + + +def test_runaway_query_is_stopped(): + endless = "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c) SELECT count(*) FROM c" + with pytest.raises(MergeError, match="秒"): + execute([sales()], endless, limits=QueryLimits(timeout_seconds=0.2)) + + +# -------------------------------------------------------------------------- +# 逐格来历：宁可不下钻，也不下钻错 +# -------------------------------------------------------------------------- + + +def _assert_traces_are_true(inputs, out): + """凡是给出来历的格，都和它指向的那个输入的那一格是同一个值。""" + by_alias = {i.alias: i for i in inputs} + found = 0 + for r, row in enumerate(out.rows): + for column in out.columns: + hit = trace_cell(out.snapshot(), r, column) + if hit is None: + continue + found += 1 + source = by_alias[hit.alias].snapshot + value = source["rows"][hit.row][source["columns"].index(hit.column)] + assert same_value(row[out.columns.index(column)], value), (r, column, hit) + return found + + +def test_direct_columns_of_a_join_trace_to_the_input_rows(): + inputs = [sales(), visits()] + out = execute(inputs, JOIN_SQL) + assert out.lineage is not None + # 2026-05-02 / S01 在会员库里是第 3 行（行号从 0 数），在门店库里是第 2 行 + hit = trace_cell(out.snapshot(), 2, "到店人数") + assert (hit.alias, hit.row, hit.column) == ("v", 3, "到店人数") + hit = trace_cell(out.snapshot(), 2, "销售额") + assert (hit.alias, hit.row, hit.column) == ("s", 2, "销售额") + assert _assert_traces_are_true(inputs, out) == 20 + + +def test_renamed_columns_still_trace(): + inputs = [sales(), visits()] + out = execute(inputs, "SELECT s.门店 AS 店, v.到店人数 人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 " + "WHERE s.日期 = '2026-05-02' ORDER BY 1") + hit = trace_cell(out.snapshot(), 1, "人数") + assert (hit.alias, hit.row, hit.column) == ("v", 2, "到店人数") + assert _assert_traces_are_true(inputs, out) == 4 + + +def test_computed_columns_have_no_trace(): + inputs = [sales(), visits()] + out = execute(inputs, "SELECT s.门店, s.订单数 * 1.0 / v.到店人数 AS 转化率 FROM s JOIN v " + "ON s.日期 = v.日期 AND s.门店 = v.门店 ORDER BY s.日期, s.门店") + assert trace_cell(out.snapshot(), 0, "转化率") is None + assert trace_cell(out.snapshot(), 0, "门店").alias == "s" + + +@pytest.mark.parametrize("sql", [ + "SELECT 日期, sum(订单数) AS 订单数 FROM s GROUP BY 日期", + "SELECT DISTINCT 门店 FROM s", + "SELECT 门店 FROM s UNION SELECT 门店 FROM v", + "SELECT 门店, (SELECT max(到店人数) FROM v) AS 峰值 FROM s", + "SELECT max(订单数) AS 订单数 FROM s", + "WITH t AS (SELECT * FROM s) SELECT 订单数 FROM t", + "SELECT 门店, 订单数, row_number() OVER (ORDER BY 订单数) AS 名次 FROM s", + "SELECT * FROM s NATURAL JOIN v", +]) +def test_shapes_outside_the_grammar_have_no_trace_at_all(sql): + out = execute([sales(), visits()], sql) + assert out.lineage is None + assert all(trace_cell(out.snapshot(), r, c) is None for r in range(len(out.rows)) for c in out.columns) + + +def test_left_join_misses_have_no_trace_on_the_missing_side(): + short = [["2026-05-01", "S01", 160]] + inputs = [sales(), visits(short)] + out = execute(inputs, "SELECT s.日期, s.门店, v.到店人数 FROM s LEFT JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 " + "ORDER BY s.日期, s.门店") + assert out.rows[1][2] is None + assert trace_cell(out.snapshot(), 1, "到店人数") is None + assert trace_cell(out.snapshot(), 1, "门店").alias == "s" + assert trace_cell(out.snapshot(), 0, "到店人数").row == 0 + _assert_traces_are_true(inputs, out) + + +def test_star_expands_in_from_order(): + inputs = [sales(), visits()] + out = execute(inputs, "SELECT * FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 ORDER BY s.日期, s.门店") + assert out.columns == [*SALES_COLS, *VISITS_COLS] + # 重名的列按第一次出现引用：「日期」指的是门店库那一列 + assert trace_cell(out.snapshot(), 0, "日期").alias == "s" + assert trace_cell(out.snapshot(), 0, "到店人数").alias == "v" + _assert_traces_are_true(inputs, out) + + +def test_using_columns_written_bare_have_no_trace(): + """USING 合并掉的列不写表名时，SQLite 取的是哪一边说不准（RIGHT / FULL JOIN 时是两边合起来的值）。""" + out = execute([sales(), visits()], "SELECT 日期, 门店, 订单数, 到店人数 FROM s JOIN v USING (日期, 门店) ORDER BY 1, 2") + assert trace_cell(out.snapshot(), 0, "日期") is None + assert trace_cell(out.snapshot(), 0, "订单数").alias == "s" + + +def test_identical_output_rows_are_not_traced(): + """两行结果一模一样时，说不清哪一行来自哪一行输入：不给。""" + twins = [["2026-05-01", "S01", 7], ["2026-05-01", "S01", 7], ["2026-05-02", "S02", 9]] + inputs = [MergeInput(alias="v", node_id="n", artifact="b" * 64, snapshot=snap(VISITS_COLS, twins))] + out = execute(inputs, "SELECT 日期, 门店, 到店人数 FROM v") + assert trace_cell(out.snapshot(), 0, "到店人数") is None and trace_cell(out.snapshot(), 1, "到店人数") is None + assert trace_cell(out.snapshot(), 2, "到店人数").row == 2 + + +def test_self_join_traces_each_side_separately(): + inputs = [visits()] + out = execute(inputs, "SELECT a.门店, a.到店人数 AS 首日, b.到店人数 AS 次日 FROM v AS a JOIN v AS b " + "ON a.门店 = b.门店 AND a.日期 = '2026-05-01' AND b.日期 = '2026-05-02' ORDER BY a.门店") + assert out.rows == [["S01", 160, 150], ["S02", 90, 100]] + first, second = trace_cell(out.snapshot(), 0, "首日"), trace_cell(out.snapshot(), 0, "次日") + assert (first.alias, first.row) == ("v", 0) and (second.alias, second.row) == ("v", 3) + _assert_traces_are_true(inputs, out) + + +def test_a_rowid_column_in_the_input_does_not_confuse_the_trace(): + rows = [[9, "S01", 1], [8, "S02", 2]] + inputs = [MergeInput(alias="t", node_id="n", artifact="b" * 64, snapshot=snap(["rowid", "门店", "值"], rows))] + out = execute(inputs, "SELECT 门店, 值 FROM t ORDER BY 门店 DESC") + assert out.rows == [["S02", 2], ["S01", 1]] + assert trace_cell(out.snapshot(), 0, "值").row == 1 + _assert_traces_are_true(inputs, out) + + +# -------------------------------------------------------------------------- +# 快照形状 +# -------------------------------------------------------------------------- + + +def test_snapshot_has_the_query_shape_plus_inputs(): + out = execute([sales(), visits()], JOIN_SQL) + snapshot = out.snapshot() + for key in ("columns", "rows", "row_count", "truncated", "sql", "column_types"): + assert key in snapshot + assert snapshot["source"] == MERGE_SOURCE + assert snapshot["inputs"] == [ + {"alias": "s", "node_id": "q_sales", "artifact": "a" * 64, "rows": 4, "columns": SALES_COLS, "source": "stores"}, + {"alias": "v", "node_id": "q_visits", "artifact": "b" * 64, "rows": 4, "columns": VISITS_COLS, + "source": "members"}, + ] + assert snapshot["sources"] == ["stores", "members"] + # 内容确定：同样的输入、同样的 SQL，快照逐字相同（重放、继续运行得到同一个工件） + assert execute([sales(), visits()], JOIN_SQL).snapshot() == snapshot + + +def test_masked_input_columns_stay_masked_even_when_renamed(): + members = MergeInput(alias="v", node_id="q_visits", artifact="b" * 64, + snapshot=snap(["门店", "手机号"], [["S01", "138****"]], source="members", + mask_columns=["手机号"])) + out = execute([sales(), members], "SELECT s.门店, v.手机号 AS 联系方式 FROM s JOIN v ON s.门店 = v.门店") + assert set(out.snapshot()["mask_columns"]) == {"手机号", "联系方式"} + + +def test_nested_merges_keep_the_underlying_sources(): + first = execute([sales(), visits()], JOIN_SQL) + nested = MergeInput(alias="m", node_id="merge1", artifact="e" * 64, snapshot=first.snapshot()) + out = execute([nested], "SELECT 日期, 门店, 到店人数 FROM m WHERE 门店 = 'S02'") + assert out.snapshot()["sources"] == ["stores", "members"] + hit = trace_cell(out.snapshot(), 0, "到店人数") + assert (hit.alias, hit.row) == ("m", 1) diff --git a/backend/tests/test_merge_node.py b/backend/tests/test_merge_node.py new file mode 100644 index 00000000..d17edc02 --- /dev/null +++ b/backend/tests/test_merge_node.py @@ -0,0 +1,498 @@ +"""合并查询节点（merge）：画图时的校验、执行、证据链和下钻、助手说明。 + +端到端用两个独立的 SQLite 夹具（tests/fixtures/merge/）：门店库的销售订单、会员库的到店记录。两条查询各自在自己的 +库里聚合到「日期 + 门店」，合并查询在库外按键合并，口径卡用 cell(nodes.merge, …) 算转化率和客单价，报告引用合并 +结果的单元格和口径卡的指标。断言报告里的数能追到两边的原始查询，下钻不指错格子。 + +不调用任何模型：报告的写作者换成脚本（monkeypatch MockChatModel._decide）。 +""" +from __future__ import annotations + +import asyncio +import json +import sqlite3 +from pathlib import Path + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage +from sqlalchemy import select + +from app.core import artifact_store +from app.core.config import settings +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource, Run, RunEvent +from app.engine.evidence import iter_segments +from app.engine.merge_query import MERGE_SOURCE +from app.engine.runner import run_manager +from app.engine.schema import GraphSpec, NodeType, validate_graph +from app.main import app + +FIXTURES = Path(__file__).parent / "fixtures" / "merge" + +SALES_SQL = ("SELECT order_date AS 日期, store_id AS 门店, COUNT(*) AS 订单数, SUM(amount) AS 销售额 FROM orders " + "GROUP BY order_date, store_id ORDER BY order_date, store_id") +# 故意和门店库排成不同的顺序：合并结果的第 0 行在会员库的结果里是第 2 行 +VISITS_SQL = ("SELECT visit_date AS 日期, store_code AS 门店, COUNT(*) AS 到店人数 FROM visits " + "GROUP BY visit_date, store_code ORDER BY visit_date DESC, store_code") +MERGE_SQL = ("SELECT s.日期, s.门店, s.订单数, s.销售额, v.到店人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 " + "ORDER BY s.日期, s.门店") +TEXT = ("5 月 1 日湖滨店到店 [[v:Q3.r0.到店人数]] 人，转化率 [[m:conversion]]，客单价 [[m:aov]]。\n\n" + "[[table:Q3 cols=日期,门店,订单数,到店人数 rows=0-3]]") + + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def _source(name: str, path: Path, options: dict | None = None) -> str: + async with SessionLocal() as session: + row = (await session.execute(select(DataSource).where(DataSource.name == name))).scalar_one_or_none() + if row is None: + row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + row.database, row.kind, row.enabled, row.readonly, row.options = str(path), "sqlite", True, True, options or {} + await session.commit() + return row.id + + +@pytest.fixture +async def sources(tmp_path, engine_up): + """把两个夹具脚本建成两个独立的 SQLite 库，各接成一个只读数据源。""" + ids = [] + for name in ("stores", "members"): + path = tmp_path / f"{name}.db" + db = sqlite3.connect(path) + db.executescript((FIXTURES / f"{name}.sql").read_text("utf-8")) + db.commit() + db.close() + ids.append(await _source(f"merge_{name}", path)) + for sid in ids: + await engines.invalidate(sid) + yield + for sid in ids: + await engines.invalidate(sid) + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def edge(a, b): + return {"source": a, "target": b} + + +def store_graph(*, sales_sql=SALES_SQL, visits_sql=VISITS_SQL, merge_sql=MERGE_SQL, visits_args=None, report=True, + inputs=None): + """入口分两路查两个库，汇合到合并查询，再接口径卡、报告、成果。""" + nodes = [ + node("start", "input"), + node("q_sales", "tool", tool="db_query__merge_stores", args={"sql": sales_sql}), + node("q_visits", "tool", tool="db_query__merge_members", args={"sql": visits_sql, **(visits_args or {})}), + node("merge", "merge", inputs=inputs or {"s": "q_sales", "v": "q_visits"}, sql=merge_sql), + node("card", "metrics", caliber="门店经营", caliber_version="v1", metrics=[ + {"id": "conversion", "name": "转化率", "decimals": 4, "format": "percent_of_ratio", + "expression": "cell(nodes.merge, 0, '订单数') / cell(nodes.merge, 0, '到店人数')"}, + {"id": "aov", "name": "客单价", "unit": "元", "decimals": 1, + "expression": "cell(nodes.merge, 0, '销售额') / cell(nodes.merge, 0, '订单数')"}]), + ] + edges = [edge("start", "q_sales"), edge("start", "q_visits"), edge("q_sales", "merge"), edge("q_visits", "merge"), + edge("merge", "card")] + if report: + nodes += [node("write", "report", instructions="写门店日报", on_violation="flag"), + node("out", "output", fields=[{"name": "日报", "value": "{{ nodes.write.text }}"}])] + edges += [edge("card", "write"), edge("write", "out")] + else: + nodes.append(node("out", "output", fields=[{"name": "转化率", "value": "{{ nodes.card.metrics }}"}])) + edges.append(edge("card", "out")) + return {"nodes": nodes, "edges": edges} + + +def script(monkeypatch, text=TEXT): + from app.providers import mock_model + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text)) + + +async def wait(run_id: str) -> Run: + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + return row + raise AssertionError(f"run {run_id} 没有结束：{row.status}") + + +async def run_graph(monkeypatch, graph=None, text=TEXT) -> Run: + script(monkeypatch, text) + return await wait((await run_manager.start(graph=graph or store_graph(), input_payload={})).id) + + +async def events(run_id: str, etype: str, node_id: str | None = None) -> list[dict]: + async with SessionLocal() as session: + query = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype) + if node_id: + query = query.where(RunEvent.node_id == node_id) + return [e.data for e in (await session.execute(query.order_by(RunEvent.seq))).scalars()] + + +def load_doc(row: Run) -> dict: + doc_id = row.output["_evidence"]["doc_artifact"] + return json.loads((settings.data_dir / "artifacts" / doc_id[:2] / f"{doc_id}.json").read_text("utf-8")) + + +def seg_of(doc: dict, text: str, n: int = 0) -> dict: + return [s for s in iter_segments(doc) if s["text"] == text and s.get("cite")][n] + + +def alias_of(doc: dict, node_id: str) -> str: + [alias] = [a for a, e in doc["catalog"].items() if e.get("kind") == "query" and e.get("node_id") == node_id] + return alias + + +# -------------------------------------------------------------------------- +# 画图时的校验 +# -------------------------------------------------------------------------- + + +def merge_spec(merge_cfg: dict, *, extra_nodes=(), extra_edges=()) -> GraphSpec: + base = store_graph(report=False) + nodes = [n if n["id"] != "merge" else node("merge", "merge", **merge_cfg) for n in base["nodes"]] + return GraphSpec.model_validate({"nodes": [*nodes, *extra_nodes], "edges": [*base["edges"], *extra_edges]}) + + +def merge_errors(spec: GraphSpec) -> list: + return [i for i in validate_graph(spec).issues if i.node_id == "merge" and i.level == "error"] + + +def test_node_type_exists(): + assert NodeType.MERGE == "merge" + + +def test_a_well_formed_merge_validates(): + spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL}) + assert merge_errors(spec) == [] + + +def test_merge_of_a_merge_is_allowed(): + spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL}, + extra_nodes=[node("again", "merge", inputs={"m": "merge"}, sql="SELECT * FROM m")], + extra_edges=[edge("merge", "again")]) + assert [i for i in validate_graph(spec).issues if i.node_id == "again" and i.level == "error"] == [] + + +def test_inputs_are_required(): + [issue] = merge_errors(merge_spec({"inputs": {}, "sql": MERGE_SQL})) + assert issue.field == "inputs" and issue.code == "merge.inputs_missing" + + +def test_input_pointing_to_a_missing_node(): + [issue] = merge_errors(merge_spec({"inputs": {"s": "q_sales", "v": "nope"}, "sql": MERGE_SQL})) + assert issue.field == "inputs.v" and "不存在" in issue.message and issue.code == "merge.input_invalid" + + +@pytest.mark.parametrize("extra, why", [ + (node("think", "llm", prompt="写点什么"), "不产出查询结果"), + (node("calc", "tool", tool="calculator", args={"expression": "1+1"}), "不是数据库查询工具"), + (node("asker", "agent", prompt="查一下", tools=["db_query__merge_stores"]), "Agent"), +]) +def test_input_must_produce_a_query_result(extra, why): + spec = merge_spec({"inputs": {"s": "q_sales", "v": extra["id"]}, "sql": MERGE_SQL}, + extra_nodes=[extra], extra_edges=[edge("start", extra["id"]), edge(extra["id"], "merge")]) + [issue] = merge_errors(spec) + assert issue.field == "inputs.v" and why in issue.message and issue.code == "merge.input_invalid" + + +def test_agent_inputs_explain_how_to_fix(): + extra = node("asker", "agent", prompt="查一下", tools=["db_query__merge_stores"]) + spec = merge_spec({"inputs": {"s": "q_sales", "v": "asker"}, "sql": MERGE_SQL}, + extra_nodes=[extra], extra_edges=[edge("start", "asker"), edge("asker", "merge")]) + [issue] = merge_errors(spec) + assert "查询多次" in issue.message and "调用工具" in issue.message + + +def test_input_must_be_upstream(): + lonely = node("q_other", "tool", tool="db_query__merge_stores", args={"sql": "SELECT 1 AS x"}) + spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_other"}, "sql": MERGE_SQL}, + extra_nodes=[lonely], extra_edges=[edge("start", "q_other")]) + [issue] = merge_errors(spec) + assert issue.field == "inputs.v" and "上游" in issue.message + + +@pytest.mark.parametrize("alias", ["1s", "a-b", "select", "sqlite_x"]) +def test_alias_must_be_a_legal_table_name(alias): + [issue] = merge_errors(merge_spec({"inputs": {"s": "q_sales", alias: "q_visits"}, "sql": MERGE_SQL})) + assert issue.field == f"inputs.{alias}" and issue.code == "merge.alias_invalid" and alias in issue.message + + +@pytest.mark.parametrize("sql, why", [ + ("", "还没有填写"), + ("DELETE FROM s", "SELECT 或 WITH"), + ("PRAGMA table_info(s)", "SELECT 或 WITH"), + ("SELECT * FROM s; SELECT * FROM v", "一条"), +]) +def test_sql_must_be_one_select_or_with(sql, why): + [issue] = merge_errors(merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": sql})) + assert issue.field == "sql" and why in issue.message and issue.code == "merge.sql_invalid" + + +def test_merge_inputs_count_as_references_to_the_upstream_nodes(): + from app.engine.variables import analyze + + report = analyze(merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL})) + refs = {v.path: v.refs for v in report.variables} + assert any(r.node_id == "merge" for r in refs["nodes.q_sales"]) + + +def test_publish_checks_treat_the_merge_as_traceable_evidence(): + """受管级别的发布检查：合并结果喂给口径卡、报告，和数据源查询一样算可追溯的证据，不在合并节点上报任何问题。""" + from app.engine.governance import publish_issues + + graph = store_graph() + for n in graph["nodes"]: + if n["id"] == "write": + n["data"]["config"].update(numbers="strict", on_violation="fail", claims="require_citation") + if n["id"] == "out": + n["data"]["config"]["contract"] = {"report_from": "write", "metrics_from": ["card"], + "required": ["conversion"], "strict": True} + issues = publish_issues(GraphSpec.model_validate(graph), level="governed") + assert [i for i in issues if i.node_id == "merge"] == [] + assert [i for i in issues if i.level == "error"] == [], [i.message for i in issues if i.level == "error"] + + +def test_labels_and_governance_know_the_type(): + from app.engine.governance import _traceable + from app.engine.labels import field_label, type_label + from app.engine.upgrade import _queries + + spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL}) + merge = spec.node_map()["merge"] + assert type_label("merge") == "合并查询" + assert field_label("sql", "merge") == "合并 SQL" and field_label("inputs", "merge") == "输入" + assert _traceable(merge) and _queries(merge) + + +# -------------------------------------------------------------------------- +# 执行与证据链 +# -------------------------------------------------------------------------- + + +async def test_end_to_end_two_databases(monkeypatch, sources, client): + row = await run_graph(monkeypatch) + assert row.status == "succeeded", row.error + + [finished] = await events(row.id, "node.finished", "merge") + output = artifact_store.load(finished["artifact"]) + assert output["columns"] == ["日期", "门店", "订单数", "销售额", "到店人数"] + assert output["rows"] == [["2026-05-01", "S01", 3, 300, 6], ["2026-05-01", "S02", 2, 150, 4], + ["2026-05-02", "S01", 2, 250, 5], ["2026-05-02", "S02", 1, 75, 4]] + assert output["source"] == MERGE_SOURCE and output["warnings"] == [] + + # 台账：一条和数据源查询同形的 query 条目，报告据此编号 + [entry] = finished["evidence"] + assert entry["kind"] == "query" and entry["source"] == MERGE_SOURCE and entry["artifact"] == output["artifact"] + assert entry["rows"] == 4 and entry["truncated"] is False + snapshot = artifact_store.load(entry["artifact"]) + assert [i["alias"] for i in snapshot["inputs"]] == ["s", "v"] + assert snapshot["sources"] == ["merge_stores", "merge_members"] + assert snapshot["sql"] == MERGE_SQL + + # 运行事件：合并的输入、SQL、行数，运行详情据此展示 + [done] = await events(row.id, "merge.end", "merge") + assert done["sql"] == MERGE_SQL and done["rows"] == 4 and done["warnings"] == [] + assert [(i["alias"], i["node_id"], i["rows"], i["source"]) for i in done["inputs"]] == [ + ("s", "q_sales", 4, "merge_stores"), ("v", "q_visits", 4, "merge_members")] + assert done["query_artifact"] == entry["artifact"] + + doc = load_doc(row) + q3 = doc["catalog"]["Q3"] + assert q3["node_id"] == "merge" and q3["source"] == MERGE_SOURCE and q3["artifact"] == entry["artifact"] + assert {alias_of(doc, "q_sales"), alias_of(doc, "q_visits")} == {"Q1", "Q2"} + metric = doc["catalog"]["m:conversion"] + assert metric["value"] == 0.5 and metric["rendered"] == "50.00%" + # 口径卡的两个输入都按 cell() 落在合并结果的那一格上，并按快照核对过 + card = artifact_store.load(metric["artifact"]) + conversion = next(m for m in card["metrics"] if m["id"] == "conversion") + assert [(i["via"], i["status"], i["artifact"], i["locator"]) for i in conversion["inputs"]] == [ + ("tool_cell", "verified", entry["artifact"], {"row": 0, "column": "订单数"}), + ("tool_cell", "verified", entry["artifact"], {"row": 0, "column": "到店人数"})] + text = row.output["日报"] + assert "到店 6 人" in text and "50.00%" in text and "100.0元" in text + + +async def test_cell_segment_traces_to_both_original_queries(monkeypatch, sources, client): + row = await run_graph(monkeypatch) + assert row.status == "succeeded", row.error + doc = load_doc(row) + qv = alias_of(doc, "q_visits") + qs = alias_of(doc, "q_sales") + seg = seg_of(doc, "6") + body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}")).json() + merge_step, visits_step = body["chain"] + + assert merge_step["step"] == "query" and merge_step["alias"] == "Q3" and merge_step["source"] == MERGE_SOURCE + assert merge_step["hash_ok"] is True and merge_step["sealed"] is True + assert "mask_note" not in merge_step # 合并查询不是数据源，不能说它「已不存在」 + merged = merge_step["merge"] + assert merged["sql"] == MERGE_SQL and merged["warnings"] == [] + assert [(i["alias"], i["node_id"], i["query"], i["rows"], i["source"], i["sealed"]) for i in merged["inputs"]] == [ + ("s", "q_sales", qs, 4, "merge_stores", True), ("v", "q_visits", qv, 4, "merge_members", True)] + assert merged["traced"] == [{"cell": [0, "到店人数"], "input": "v", "query": qv, "row": 2, "column": "到店人数"}] + + # 接着给出会员库那次查询，高亮的正是那一格，值相同 + assert visits_step["step"] == "query" and visits_step["alias"] == qv and visits_step["merged_into"] == "Q3" + assert visits_step["highlight"]["cells"] == [[2, "到店人数"]] + at = visits_step["row_index"].index(2) + assert visits_step["rows"][at][visits_step["columns"].index("到店人数")] == 6 + + +async def test_metric_segment_traces_through_the_merge(monkeypatch, sources, client): + row = await run_graph(monkeypatch) + doc = load_doc(row) + qs, qv = alias_of(doc, "q_sales"), alias_of(doc, "q_visits") + body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_of(doc, '50.00%')['id']}")).json() + metric, *inputs = body["chain"][:3] + assert metric["step"] == "metric" and metric["value"] == 0.5 + assert [(i["via"], i["status"], i["query"], i["cell"]) for i in inputs] == [ + ("tool_cell", "verified", "Q3", "Q3.r0.订单数"), ("tool_cell", "verified", "Q3", "Q3.r0.到店人数")] + steps = [s for s in body["chain"] if s["step"] == "query"] + assert [s["alias"] for s in steps] == ["Q3", qs, qv] + merge_step, sales_step, visits_step = steps + assert sorted(t["input"] for t in merge_step["merge"]["traced"]) == ["s", "v"] + assert sales_step["highlight"]["cells"] == [[0, "订单数"]] and sales_step["merged_into"] == "Q3" + assert visits_step["highlight"]["cells"] == [[2, "到店人数"]] + + +async def test_provenance_of_a_merged_cell_follows_the_trace(monkeypatch, sources, client): + """手工源上的格：追到会员库那次查询，结论和直接引用那一格相同（not_upload），并记下经过的合并。""" + row = await run_graph(monkeypatch) + doc = load_doc(row) + seg = seg_of(doc, "6") + out = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}/provenance")).json() + assert (out["status"], out["reason"]["code"]) == ("none", "not_upload") + assert out["cell"] == {"alias": "Q3", "row": 0, "column": "到店人数", "artifact": doc["catalog"]["Q3"]["artifact"]} + assert out["merge"] == [{"alias": "Q3", "node_id": "merge", "input": "v", "query": alias_of(doc, "q_visits"), + "row": 2, "column": "到店人数"}] + + +async def test_a_computed_merge_column_has_no_cell_lineage(monkeypatch, sources, client): + sql = ("SELECT s.日期, s.门店, s.订单数 + v.到店人数 AS 合计 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 " + "ORDER BY s.日期, s.门店") + graph = store_graph(merge_sql=sql) + graph["nodes"] = [n for n in graph["nodes"] if n["id"] != "card"] + graph["edges"] = [e for e in graph["edges"] if "card" not in (e["source"], e["target"])] + [edge("merge", "write")] + row = await run_graph(monkeypatch, graph, text="合计 [[v:Q3.r0.合计]]。") + assert row.status == "succeeded", row.error + doc = load_doc(row) + seg = seg_of(doc, "9") + body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}")).json() + [merge_step] = body["chain"] + [traced] = merge_step["merge"]["traced"] + assert traced["input"] is None and traced["query"] is None and "逐格来历" in traced["note"] + out = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}/provenance")).json() + assert (out["status"], out["reason"]["code"], out["version"]) == ("none", "merge_no_lineage", None) + assert out["merge"] == [{"alias": "Q3", "node_id": "merge", "input": None, "query": None, "row": None, + "column": None}] + + +async def test_evidence_graph_links_the_merge_to_its_inputs(monkeypatch, sources, client): + row = await run_graph(monkeypatch) + doc = load_doc(row) + body = (await client.get(f"/api/runs/{row.id}/evidence")).json() + merged = sorted((e["to"], e["input"]) for e in body["edges"] if e["rel"] == "merged_from" and e["from"] == "Q3") + assert merged == sorted([(alias_of(doc, "q_sales"), "s"), (alias_of(doc, "q_visits"), "v")]) + + +def test_judge_excerpt_names_the_inputs_and_masks_their_sources(): + """裁判看合并结果时，知道合并了哪几个输入；输入数据源现在设的遮罩照样生效。""" + from app.engine import judge + from app.engine.merge_query import MergeInput, execute + + sales = {"columns": ["门店", "订单数"], "rows": [["S01", 3]], "truncated": False, "source": "merge_stores"} + visits = {"columns": ["门店", "手机号"], "rows": [["S01", "13800000000"]], "truncated": False, + "source": "merge_members"} + snap = execute([MergeInput("s", "q_sales", "a" * 64, sales), MergeInput("v", "q_visits", "b" * 64, visits)], + "SELECT s.门店, s.订单数, v.手机号 FROM s JOIN v ON s.门店 = v.门店").snapshot() + entry = {"kind": "query", "artifact": "m" * 64, "source": MERGE_SOURCE} + text = judge._query_excerpt("Q3", entry, {0}, ["订单数"], False, lambda _a: snap, + {"merge_members": ["手机号"]}) + assert text.startswith("【Q3】合并查询结果（共 1 行）") + assert "合并的输入：s（节点 q_sales，数据源 merge_stores，1 行）、v（节点 q_visits，数据源 merge_members，1 行）" in text + assert "13800000000" not in text and "遮罩" in text + + +async def test_a_truncated_input_fails_the_node(monkeypatch, sources): + row = await run_graph(monkeypatch, store_graph(visits_args={"limit": 2}, report=False)) + assert row.status == "failed" + [failed] = await events(row.id, "node.failed", "merge") + assert "「v」" in failed["error"] and "截断" in failed["error"] and "聚合" in failed["error"] + + +async def test_warnings_reach_the_output_and_the_events(monkeypatch, sources): + """门店编号一边是文本 '01'、一边是数 1；只按门店合并、没按日期：两道警告都要看得见。""" + sales_sql = ("SELECT order_date AS 日期, substr(store_id, 2) AS 门店, COUNT(*) AS 订单数, SUM(amount) AS 销售额 " + "FROM orders GROUP BY order_date, store_id ORDER BY 1, 2") + visits_sql = ("SELECT visit_date AS 日期, CAST(substr(store_code, 2) AS INTEGER) AS 门店, COUNT(*) AS 到店人数 " + "FROM visits GROUP BY visit_date, store_code ORDER BY 1, 2") + merge_sql = "SELECT s.门店, s.订单数, s.销售额, v.到店人数 FROM s JOIN v ON s.门店 = v.门店" + row = await run_graph(monkeypatch, store_graph(sales_sql=sales_sql, visits_sql=visits_sql, merge_sql=merge_sql, + report=False)) + assert row.status == "succeeded", row.error + [finished] = await events(row.id, "node.finished", "merge") + output = artifact_store.load(finished["artifact"]) + assert sorted(w["code"] for w in output["warnings"]) == ["key_type_mismatch", "rows_grew"] + [done] = await events(row.id, "merge.end", "merge") + assert sorted(w["code"] for w in done["warnings"]) == ["key_type_mismatch", "rows_grew"] + logs = [e for e in await events(row.id, "log", "merge") if e.get("level") == "warn"] + assert sorted(e["code"] for e in logs) == ["merge_key_type", "merge_rows_grew"] + assert all(e["message"] for e in logs) + + +async def test_rerunning_the_node_on_the_same_inputs_gives_the_same_snapshot(monkeypatch, sources): + """重放、继续运行时节点从头再执行：同样的输入得到同一份快照、同一条台账，下游的出处不变。""" + from app.engine.context import NodeContext, RunContext + from app.engine.nodes.merge import run_merge + + row = await run_graph(monkeypatch, store_graph(report=False)) + assert row.status == "succeeded", row.error + outputs = {} + for nid in ("q_sales", "q_visits"): + [finished] = await events(row.id, "node.finished", nid) + outputs[nid] = artifact_store.load(finished["artifact"]) + spec = GraphSpec.model_validate(store_graph(report=False)) + ctx = NodeContext(node=spec.node_map()["merge"], + run=RunContext(run_id=row.id, thread_id=row.id, spec=spec, agent_limits={})) + state = {"nodes": outputs, "trail": [], "evidence": []} + first, second = await run_merge(state, ctx), await run_merge(state, ctx) + assert first == second + assert first["evidence"][0]["artifact"] == first["nodes"]["merge"]["artifact"] + + +# -------------------------------------------------------------------------- +# 助手 +# -------------------------------------------------------------------------- + + +def test_node_reference_explains_when_to_use_merge(): + from app.api.copilot import NODE_REFERENCE + + assert "- merge：" in NODE_REFERENCE + block = NODE_REFERENCE.split("- merge：", 1)[1].split("\n- ", 1)[0] + for phrase in ("inputs", "sql", "不同的库", "同一个库", "一条 SQL", "口径卡", "截断", "cell(nodes."): + assert phrase in block, phrase diff --git a/backend/tests/test_merge_provenance.py b/backend/tests/test_merge_provenance.py new file mode 100644 index 00000000..f90c72c2 --- /dev/null +++ b/backend/tests/test_merge_provenance.py @@ -0,0 +1,113 @@ +"""合并查询的结果接进证据下钻（期 4「推断的来源」）：宁可不下钻，也不下钻错。 + +上传表格（合成客流表，按配方导入，夹具同 test_provenance_api）查一次、手工源的气温快照现造一份，两者在库外按日期 +合并。合并结果里直接来自客流表的格，下钻结论要和直接引用客流表那一格完全相同（同一个原表格子）；合并 SQL 算出来的 +列判为没有逐格来历；来自手工源的格照手工源的结论（not_upload）。每份结论都断言 contract_problems 为空。 +""" +from __future__ import annotations + +import pytest + +from app.api import evidence as api +from app.core import artifact_store +from app.data.provenance_types import DOC_PROVENANCE, MergeHop +from app.engine.merge_query import MERGE_SOURCE, MergeInput, execute +from tests.fixtures.xlsx.flow import SHEET +from tests.test_provenance_api import ( # noqa: F401 夹具按名字注入（rep 依赖 upload_world、client） + cell_seg, + cell_value, + client, + ok, + put, + rep, + report_of, + run_sql, + upload_world, +) + +FLOW_SQL = 'SELECT "日期", "全日客流" FROM "日客流" ORDER BY "日期" DESC LIMIT 6' +MERGE_SQL = ('SELECT f."日期", f."全日客流", w."气温", f."全日客流" * 2 AS 加倍 FROM f JOIN w ON f."日期" = w."日期" ' + 'ORDER BY f."日期"') + + +async def merged_world(up, c): + """(文档, 封存范围, 合并结果, 客流快照)。客流按日期倒序查，合并结果按日期正序排：合并结果第 0 行是客流第 5 行。""" + qid, sid = await run_sql(c, up, FLOW_SQL) + flow = artifact_store.load(qid) + dates = sorted(r[0] for r in flow["rows"]) + weather = {"columns": ["日期", "气温"], "rows": [[d, 20 + i] for i, d in enumerate(dates)], "row_count": len(dates), + "truncated": False, "sql": "SELECT 日期, 气温 FROM 每日气温", "source": "weather_demo", + "column_types": {"日期": "text", "气温": "number"}} + wid = await put(weather) + outcome = execute([MergeInput(alias="f", node_id="q_flow", artifact=qid, snapshot=flow), + MergeInput(alias="w", node_id="q_weather", artifact=wid, snapshot=weather)], MERGE_SQL) + mid = await put(outcome.snapshot()) + doc = {"schema": "agentlab.report/1", "entity_syntax": 2, "run_id": "run-merge", "node_id": "write", + "provenance": DOC_PROVENANCE, "blocks": [], "catalog": { + "Q1": {"kind": "query", "artifact": qid, "tool": f"db_query__{up.name}", "node_id": "q_flow", + "source": up.name}, + "Q2": {"kind": "query", "artifact": wid, "tool": "db_query__weather_demo", "node_id": "q_weather", + "source": "weather_demo"}, + "Q3": {"kind": "query", "artifact": mid, "node_id": "merge", "source": MERGE_SOURCE}}} + seal = {"sealed": True, "ok": True, "manifest_seq": 9, "legacy": False} + sealed = api._Sealed(run_id="run-merge", graph={}, seal=seal, events=[], queries={ + qid: {"node_id": "q_flow", "tool": f"db_query__{up.name}"}, + wid: {"node_id": "q_weather", "tool": "db_query__weather_demo"}, + mid: {"node_id": "merge", "source": MERGE_SOURCE}}, schemas={sid: {"node_id": "q_flow"}}) + return doc, sealed, outcome, flow + + +async def drill(doc, sealed, *, alias, row, column): + return ok(await api.segment_provenance(report_of(doc), cell_seg(alias=alias, row=row, column=column), sealed, + masks=api._Masks())) + + +async def test_a_merged_upload_cell_points_at_the_same_original_cell(rep, client): + doc, sealed, outcome, flow = await merged_world(rep, client) + assert len(outcome.rows) == 6 + for r in range(len(outcome.rows)): + via_merge = await drill(doc, sealed, alias="Q3", row=r, column="全日客流") + direct = await drill(doc, sealed, alias="Q1", row=5 - r, column="全日客流") + assert via_merge.status == direct.status == "inferred" + # 同一个原表格子、同一份数据版本：结论逐字相同，只是 cell 仍是报告引用的那一格，并记下经过的合并 + assert via_merge.cell_source == direct.cell_source and via_merge.version == direct.version + assert (via_merge.cell.alias, via_merge.cell.row, via_merge.cell.column) == ("Q3", r, "全日客流") + assert via_merge.merge == [MergeHop(alias="Q3", node_id="merge", input="f", query="Q1", row=5 - r, + column="全日客流")] + # 原表那一格的值就是合并结果里这一格的值 + assert cell_value(rep.raws[0], SHEET, via_merge.cell_source.cell) == outcome.rows[r][1] + + +async def test_a_computed_merge_column_is_not_drilled(rep, client): + doc, sealed, _outcome, _flow = await merged_world(rep, client) + out = await drill(doc, sealed, alias="Q3", row=0, column="加倍") + assert (out.status, out.reason.code, out.version, out.cell_source) == ("none", "merge_no_lineage", None, None) + assert out.merge == [MergeHop(alias="Q3", node_id="merge")] + assert "表级来历" in out.reason.text + + +async def test_a_manual_source_column_keeps_its_own_answer(rep, client): + doc, sealed, _outcome, _flow = await merged_world(rep, client) + out = await drill(doc, sealed, alias="Q3", row=0, column="气温") + assert (out.status, out.reason.code) == ("none", "not_upload") + assert out.merge == [MergeHop(alias="Q3", node_id="merge", input="w", query="Q2", row=0, column="气温")] + + +async def test_an_unsealed_merge_snapshot_is_not_drilled(rep, client): + doc, sealed, _outcome, _flow = await merged_world(rep, client) + sealed.queries.pop(doc["catalog"]["Q3"]["artifact"]) + out = await drill(doc, sealed, alias="Q3", row=0, column="全日客流") + assert (out.status, out.reason.code) == ("none", "not_sealed") + + +@pytest.mark.parametrize("column, hinted", [("全日客流", True), ("气温", False), ("加倍", False)]) +async def test_the_query_step_hint_follows_the_trace(rep, client, column, hinted): + """界面只在追得到上传表格的格上请求推断来源；合并步骤后面接上追到的输入步骤。""" + doc, sealed, _outcome, _flow = await merged_world(rep, client) + steps = await api._query_steps(doc["catalog"]["Q3"]["artifact"], [(0, column)], doc, sealed, api._Masks(), + hint=True) + assert steps[0]["merge"]["inputs"][0]["query"] == "Q1" + assert ("provenance" in steps[0]) is hinted + assert len(steps) == (1 if column == "加倍" else 2) + if len(steps) == 2: + assert steps[1]["merged_into"] == "Q3" and "provenance" not in steps[1] diff --git a/backend/tests/test_provenance_contract.py b/backend/tests/test_provenance_contract.py index 730eae52..806e0953 100644 --- a/backend/tests/test_provenance_contract.py +++ b/backend/tests/test_provenance_contract.py @@ -44,7 +44,7 @@ def test_literals(): "legacy_doc", "not_cell", "not_sealed", "not_upload", "simple_upload", "manifest_unreadable", "chain_mismatch", "expression", "alias", "multi_table", "unparsed", "no_pk", "pk_missing", "masked", "null_value", "null_pk", "snapshot_gone", "db_tampered", "recheck_missing", "recheck_multiple", - "recheck_mismatch", "no_lineage"} + "recheck_mismatch", "no_lineage", "merge_no_lineage"} assert len(get_args(P.ReasonCode)) == len(set(get_args(P.ReasonCode))) assert set(get_args(P.AlertCode)) == {"db_tampered", "chain_mismatch", "manifest_unreadable"} assert set(get_args(P.AlertCode)) <= set(get_args(P.ReasonCode)) @@ -130,7 +130,10 @@ SHAPES: dict[str, list[tuple[str, object]]] = { ("cell_status", None), ("acceptance", None), ("detail", None)], "ProvenanceOut": [("schema", P.SCHEMA), ("report", REQ), ("segment", REQ), ("cell", None), ("status", REQ), ("reason", None), ("alert", None), ("sealed", REQ), ("version", None), - ("cell_source", None), ("checks", FACTORY)], + ("cell_source", None), ("checks", FACTORY), ("merge", FACTORY)], + # 合并查询结果里的格经过的每一次合并（P3：合并查询节点） + "MergeHop": [("alias", REQ), ("node_id", None), ("input", None), ("query", None), ("row", None), + ("column", None)], # 中间结构（P4-SPEC 7.2–7.4） "DirectSelect": [("table", REQ), ("alias", REQ), ("columns", REQ)], "SelectRefusal": [("code", REQ), ("detail", REQ)], @@ -146,7 +149,7 @@ SHAPES: dict[str, list[tuple[str, object]]] = { } RESPONSE = ("ReportRef", "CellRef", "Reason", "Alert", "TableRef", "PeriodView", "AcceptanceView", "StateNote", "PartView", "VersionView", "FromCell", "YearSource", "CanonicalView", "Recheck", "CellSource", - "RelatedCheck", "ProvenanceOut") + "RelatedCheck", "ProvenanceOut", "MergeHop") @pytest.mark.parametrize("name", sorted(SHAPES)) @@ -184,7 +187,8 @@ def test_intermediate_types_take_positional_args_as_the_spec_writes_them(): KEYS = { "top": ["schema", "report", "segment", "cell", "status", "reason", "alert", "sealed", "version", "cell_source", - "checks"], + "checks", "merge"], + "merge": ["alias", "node_id", "input", "query", "row", "column"], "report": ["node_id", "doc_artifact"], "cell": ["alias", "row", "column", "artifact"], "reason": ["code", "detail", "text"], @@ -248,6 +252,7 @@ def full_out() -> P.ProvenanceOut: checks=[P.RelatedCheck(id="R1", kind="relation_sum_eq", title="「日客流」：全日客流 = 分区甲 + 分区乙", part_status="mismatch", row_status="passed", cell_status=None, acceptance=_accept(), detail="合成的原因摘要")], + merge=[P.MergeHop(alias="Q3", node_id="merge", input="f", query="Q1", row=5, column="全日客流")], ) @@ -258,7 +263,7 @@ def test_asdict_matches_the_spec_json_level_by_level(): "top": d, "report": d["report"], "cell": d["cell"], "version": v, "table": v["tables"][0], "part": part, "period": part["period"], "acceptance": part["acceptances"][0], "state": part["purged"], "cell_source": cs, "from": cs["from"][0], "year": cs["year"], "canonical": cs["canonical"], - "recheck": cs["recheck"], "check": d["checks"][0], + "recheck": cs["recheck"], "check": d["checks"][0], "merge": d["merge"][0], } for name, obj in levels.items(): assert isinstance(obj, dict), name @@ -280,7 +285,8 @@ def test_null_and_empty_defaults_match_the_spec(): reason=P.refuse("legacy_doc"), sealed=False) d = asdict(out) assert d["schema"] == "agentlab.provenance/1" - assert (d["cell"], d["alert"], d["version"], d["cell_source"], d["checks"]) == (None, None, None, None, []) + assert (d["cell"], d["alert"], d["version"], d["cell_source"], d["checks"], d["merge"]) == ( + None, None, None, None, [], []) assert d["reason"] == {"code": "legacy_doc", "detail": "", "text": P.REASON_TEXT["legacy_doc"]} @@ -666,7 +672,7 @@ TS_INTERFACES = { "ProvenancePeriod": "PeriodView", "ProvenanceAcceptance": "AcceptanceView", "ProvenanceStateNote": "StateNote", "ProvenancePart": "PartView", "ProvenanceVersion": "VersionView", "ProvenanceFromCell": "FromCell", "ProvenanceYear": "YearSource", "ProvenanceCanonical": "CanonicalView", "ProvenanceRecheck": "Recheck", - "ProvenanceCellSource": "CellSource", "ProvenanceCheck": "RelatedCheck", + "ProvenanceCellSource": "CellSource", "ProvenanceCheck": "RelatedCheck", "ProvenanceMergeHop": "MergeHop", } TS_UNIONS = { "ProvenanceStatus": P.Status, "ProvenanceReasonCode": P.ReasonCode, "ProvenanceAlertCode": P.AlertCode, diff --git a/frontend/src/canvas/Inspector.tsx b/frontend/src/canvas/Inspector.tsx index bf5140e5..0aeaede1 100644 --- a/frontend/src/canvas/Inspector.tsx +++ b/frontend/src/canvas/Inspector.tsx @@ -1,4 +1,4 @@ -import { useEffect, useId, useMemo, useState } from 'react' +import { useEffect, useId, useMemo, useRef, useState } from 'react' import { createPortal } from 'react-dom' import { Link } from 'react-router-dom' import { AlertTriangle, ChevronLeft, Copy, Lock, Maximize2, Plus, Trash2, X, XCircle } from 'lucide-react' @@ -12,7 +12,7 @@ import { datasourceTools, modelOptions, providerOfModel, useCatalog, useDatasour import { api } from '../api/client' import { IconButton, JsonInput, Modal, isComposing, useRadioGroup } from '../components/ui' import { formatShortcut } from '../lib/keys' -import { JUDGE_FIELD_LABEL, JUDGE_ON_UNSUPPORTED_HINT, JUDGE_ON_UNSUPPORTED_LABEL, JUDGE_SETTING_TEXT, SUBGRAPH_UPGRADE_HELP, judgeByModelText, upgradeNewerText } from '../lib/terms' +import { JUDGE_FIELD_LABEL, JUDGE_ON_UNSUPPORTED_HINT, JUDGE_ON_UNSUPPORTED_LABEL, JUDGE_SETTING_TEXT, MERGE_TEXT, NODE_TYPE_LABEL, SUBGRAPH_UPGRADE_HELP, judgeByModelText, upgradeNewerText } from '../lib/terms' import { TemplateText } from './TemplateText' import type { NodeType, ValidationIssue, WorkflowVersion } from '../types' @@ -150,6 +150,140 @@ function NodeRefsInput({ nodeId, type, value, onChange }: { ) } +interface MergeRow { alias: string; node: string } + +/** config.inputs（{别名: 节点 id}）→ 行。对象的键顺序就是行的顺序 */ +function mergeRowsOf(value: any): MergeRow[] { + if (!value || typeof value !== 'object' || Array.isArray(value)) return [] + return Object.entries(value).map(([alias, node]) => ({ alias, node: typeof node === 'string' ? node : '' })) +} + +/** 和后端 engine/merge_query.alias_problem 的写法规则一致；关键字、sqlite_ 前缀这类由后端校验报出来 */ +const MERGE_ALIAS = /^[A-Za-z_][A-Za-z0-9_]{0,31}$/ + +/** 新加一行时的别名：从节点 id 取（q_sales → q_sales），不合法的退成 t1、t2…，和已有的不重名 */ +function freshAlias(nodeId: string, taken: Set): string { + const base = nodeId.toLowerCase().replace(/[^a-z0-9_]/g, '_').replace(/^(?=[0-9])/, 't') + const head = MERGE_ALIAS.test(base) ? base : 't' + if (head !== 't' && !taken.has(head)) return head + for (let n = 1; ; n += 1) if (!taken.has(`${head}${n}`)) return `${head}${n}` +} + +/** + * 合并查询的输入：每一行一个别名（合并 SQL 里的表名）加一个上游查询节点。 + * + * 只列上游的查询节点：选了数据库查询工具的「调用工具」节点和别的合并查询。Agent 查过的库、模型写的文字都不能合并 + * （后端校验会报错），这里干脆不给选。已经选了、后来被删掉或挪到下游的节点照样显示成一行并说明，好让人看见并改掉。 + * + * 行先记在本地再写回：别名是对象的键，改到和另一行重名的那一刻写回去会吞掉一行。重名时先不写回，行内说明； + * 撤销、助手改图这类外部改动到达时按新的配置重来。后端按 inputs.<别名> 报的问题落在那一行下面 + */ +function MergeInputList({ nodeId, value, issues, onChange }: { + nodeId: string; value: any; issues: FieldIssue[]; onChange: (v: any) => void +}) { + const nodes = useStudio((s) => s.nodes) + const edges = useStudio((s) => s.edges) + const uid = useId() + const options = useMemo(() => { + const parents = new Map() + for (const e of edges) parents.set(e.target, [...(parents.get(e.target) ?? []), e.source]) + const seen = new Set() + const stack = [...(parents.get(nodeId) ?? [])] + while (stack.length) { + const id = stack.pop()! + if (seen.has(id) || id === nodeId) continue + seen.add(id) + stack.push(...(parents.get(id) ?? [])) + } + return nodes + .filter((n) => { + if (!seen.has(n.id)) return false + const tool = String((n.data?.config as Record | undefined)?.tool ?? '') + return n.data?.nodeType === 'merge' || (n.data?.nodeType === 'tool' && tool.startsWith('db_query__')) + }) + .map((n) => { + const label = String(n.data?.label || n.id) + const tool = String((n.data?.config as Record | undefined)?.tool ?? '') + return { + value: n.id, label: label === n.id ? label : `${label}（${n.id}）`, + hint: n.data?.nodeType === 'merge' ? NODE_TYPE_LABEL.merge : tool, + } + }) + }, [nodes, edges, nodeId]) + + const external = useMemo(() => mergeRowsOf(value), [value]) + const [rows, setRows] = useState(external) + const wrote = useRef(JSON.stringify(external)) + useEffect(() => { + const sig = JSON.stringify(external) + if (sig !== wrote.current) { + wrote.current = sig + setRows(external) + } + }, [external]) + const keyOf = (alias: string) => alias.toLowerCase() + const clash = (list: MergeRow[], i: number) => + !!list[i].alias && list.some((r, j) => j !== i && keyOf(r.alias) === keyOf(list[i].alias)) + const commit = (next: MergeRow[]) => { + setRows(next) + if (next.some((_, i) => clash(next, i))) return + const obj = Object.fromEntries(next.map((r) => [r.alias, r.node])) + wrote.current = JSON.stringify(mergeRowsOf(obj)) + onChange(obj) + } + const add = () => { + const used = new Set(rows.map((r) => r.node)) + const pick = options.find((o) => !used.has(o.value)) ?? options[0] + if (!pick) return + commit([...rows, { alias: freshAlias(pick.value, new Set(rows.map((r) => keyOf(r.alias)))), node: pick.value }]) + } + + return ( +
+ {rows.map((row, i) => { + const own = issues.filter((x) => x.at?.sub != null && x.at.sub === row.alias) + const local = clash(rows, i) ? `别名「${row.alias}」与另一个输入重复（不区分大小写），请换一个` + : !row.alias ? '请填写别名' + : !MERGE_ALIAS.test(row.alias) ? '别名只能用英文字母、数字和下划线，不能以数字开头，最多 32 个字符' : '' + const known = options.find((o) => o.value === row.node) + const tone = own.some((x) => x.level === 'error') || local ? 'error' : own.length ? 'warning' : undefined + return ( + commit(rows.filter((_, j) => j !== i))}> +
+ + commit(rows.map((r, j) => (j === i ? { ...r, alias: e.target.value.trim() } : r)))} /> + + + + +
+ {known?.hint &&
{known.hint}
} + {local && !own.length && ( +
+ + {local} +
+ )} + {own.map((x, k) => )} + + ) + })} + + {!options.length &&
{MERGE_TEXT.noUpstream}
} +
+ ) +} + /** * 子工作流钉住版本后的升版处置：和口径卡的是同一个下拉——直接借口径卡的字段定义，选项、 * 文案一字不差，只换说明里「和谁同一套」那半句。上游在钉住的那一版之后又发了版时，正式运行 @@ -271,7 +405,10 @@ export function Inspector() { placeholder={def.label} />
- ID {node.id} · 下游用 {`{{ nodes.${node.id}.text }}`} 引用 + {/* 合并查询的产出是一张表、没有 text：口径卡按格取值，报告按查询编号引用 */} + ID {node.id} · {node.data.nodeType === 'merge' + ? <>口径卡用 {`cell(nodes.${node.id}, 0, '列名')`} 取值 + : <>下游用 {`{{ nodes.${node.id}.text }}`} 引用}
@@ -404,8 +541,11 @@ function Field({ field, nodeId, value, config, issues, onChange }: { const syntax = syntaxOf(field) const composite = field.type === 'cases' || field.type === 'agents' || field.type === 'ioFields' || field.type === 'metricsList' || field.type === 'caliberFrom' || field.type === 'judge' - // 复合字段自己把问题落到第几项；落不到具体某一项的，和普通字段一样挂在下面。结论句裁判按子键（judge.max_cost_usd）落 + || field.type === 'mergeInputs' + // 复合字段自己把问题落到第几项；落不到具体某一项的，和普通字段一样挂在下面。结论句裁判按子键（judge.max_cost_usd）落， + // 合并查询的输入按别名（inputs.<别名>）落到那一行 const own = field.type === 'judge' ? issues.filter((i) => !i.at?.sub || !JUDGE_SUB_KEYS.has(i.at.sub)) + : field.type === 'mergeInputs' ? issues.filter((i) => !i.at?.sub) : composite ? issues.filter((i) => i.at?.index == null) : issues const bad = own.some((i) => i.level === 'error') const errorId = own.length ? `${id}-issues` : undefined @@ -606,6 +746,9 @@ function FieldInput({ field, id, nodeId, syntax, value, config, invalid, describ case 'nodeRefs': return + case 'mergeInputs': + return + case 'caliberFrom': return diff --git a/frontend/src/canvas/InspectorSheet.tsx b/frontend/src/canvas/InspectorSheet.tsx index 41bfd53a..4dce4062 100644 --- a/frontend/src/canvas/InspectorSheet.tsx +++ b/frontend/src/canvas/InspectorSheet.tsx @@ -67,7 +67,8 @@ export function revealField(nodeId: string, field?: FieldRef | string | null, op const sheet = document.querySelector('[data-inspector-sheet]') ?? document const box = sheet.querySelector(`[data-field="${CSS.escape(at.key)}"]`) const item = at.index != null ? box?.querySelector(`[data-item="${at.index}"]`) : null - const part = at.sub ? item?.querySelector(`[data-sub="${CSS.escape(at.sub)}"]`) : null + // 只有子键没有序号的（合并查询的 inputs.<别名>）：在整个字段里按子键找那一行 + const part = at.sub ? (item ?? box)?.querySelector(`[data-sub="${CSS.escape(at.sub)}"]`) : null const target = part ?? item ?? box if (!target) return const still = window.matchMedia('(prefers-reduced-motion: reduce)').matches diff --git a/frontend/src/canvas/NodeCard.tsx b/frontend/src/canvas/NodeCard.tsx index 9526c5db..5b035188 100644 --- a/frontend/src/canvas/NodeCard.tsx +++ b/frontend/src/canvas/NodeCard.tsx @@ -52,6 +52,13 @@ function summarize(type: NodeType, config: Record): string { return config.tool ? String(config.tool) : '未选择工具' case 'code': return `${config.language ?? 'python'} · ${(config.code ?? '').split('\n')[0].slice(0, 48) || '空'}` + case 'merge': { + // 别名就是合并 SQL 里的表名：卡片上看得出合并的是哪几张 + const aliases = config.inputs && typeof config.inputs === 'object' && !Array.isArray(config.inputs) + ? Object.keys(config.inputs) : [] + if (!aliases.length) return '未配置输入' + return `合并 ${aliases.join('、')}${String(config.sql ?? '').trim() ? '' : ' · 未填写合并 SQL'}` + } case 'branch': return (config.cases ?? []).map((c: any) => c.key).filter(Boolean).join(' / ') || '未配置分支' case 'loop': diff --git a/frontend/src/canvas/nodeDefs.ts b/frontend/src/canvas/nodeDefs.ts index 9e8c8e4b..f9389894 100644 --- a/frontend/src/canvas/nodeDefs.ts +++ b/frontend/src/canvas/nodeDefs.ts @@ -1,14 +1,14 @@ import { - Bot, Braces, Brain, CheckCircle2, Code2, Database, FileInput, FileOutput, FileText, + Bot, Braces, Brain, CheckCircle2, Code2, Combine, Database, FileInput, FileOutput, FileText, Gauge, GitBranch, Hand, Repeat, Search, Shuffle, Users, Wrench, } from 'lucide-react' -import { APPROVAL_POLICY_LABEL, NODE_TYPE_LABEL, UPGRADE_POLICY_LABEL } from '../lib/terms' +import { APPROVAL_POLICY_LABEL, MERGE_TEXT, NODE_TYPE_LABEL, UPGRADE_POLICY_LABEL } from '../lib/terms' import type { NodeType } from '../types' export type FieldType = | 'text' | 'textarea' | 'prompt' | 'code' | 'number' | 'select' | 'switch' | 'json' | 'model' | 'tools' | 'skills' | 'collection' - | 'ioFields' | 'cases' | 'agents' | 'metricsList' | 'nodeRefs' | 'caliberFrom' | 'judge' + | 'ioFields' | 'cases' | 'agents' | 'metricsList' | 'nodeRefs' | 'caliberFrom' | 'judge' | 'mergeInputs' /** * 这个字段里写的是什么语法。 @@ -296,6 +296,23 @@ export const NODE_DEFS: Record = { ], defaults: { args: {} }, }, + merge: { + type: 'merge', label: NODE_TYPE_LABEL.merge, category: '执行', icon: Combine, + description: '把不同数据库的查询结果按合并键合并成一张表，报告和口径卡照常引用、可查看出处', + hasTarget: true, sources: [{ id: 'out', label: '' }], + fields: [ + // 后端 config.inputs 是 {别名: 节点 id}：别名就是内存库里的表名（engine/merge_query.alias_problem 校验） + { key: 'inputs', label: '输入', type: 'mergeInputs', help: MERGE_TEXT.inputsHint }, + { + key: 'sql', label: '合并 SQL', type: 'code', + placeholder: 'SELECT s.day, s.store, s.orders, v.visitors FROM s JOIN v ON s.day = v.day AND s.store = v.store', + help: '一条 SQLite 的 SELECT 或 WITH 查询，表名用上面的别名。同一个库的数据请直接写成一条查询；' + + '比率、增幅等派生计算请写在口径卡中。直接选取输入的列（可用 AS 改名）时，报告中的数字可以追到输入的那一格', + }, + ...COMMON_TAIL, + ], + defaults: { inputs: {}, sql: '' }, + }, code: { type: 'code', label: NODE_TYPE_LABEL.code, category: '执行', icon: Code2, description: '在 microVM 或系统沙箱中执行代码', diff --git a/frontend/src/dev/preview.tsx b/frontend/src/dev/preview.tsx index 2b484041..5743eb8c 100644 --- a/frontend/src/dev/preview.tsx +++ b/frontend/src/dev/preview.tsx @@ -14,7 +14,7 @@ import evidenceFixture from '../run/__tests__/evidence-doc.json' import evidenceJudge from '../run/__tests__/evidence-judge.json' import { CALIBER_ONE_OUTPUT, CALIBERS_OUTPUT, COPILOT_STUCK, COPILOT_TOOLS_DROPPED, MIXED_OUTPUT, abandonedRun, cancelledRun, exhaustedTeam, longLoop, - markupRun, mixedRun, pipelineRun, repairRun, teamRun, timeoutRun, + markupRun, mergeRun, mixedRun, pipelineRun, repairRun, teamRun, timeoutRun, } from '../run/__tests__/synthetic' import '../index.css' @@ -348,6 +348,10 @@ function synthetic(name: string): StreamTurn[] { columns: ['factory_code', 'line_name', 'attribute_group', 'output_qty'], rows: [['1063', '一号线', '001', 1520], ['1064', '二号线', '002', 1310], ['1065', '三号线', '001', 980]], row_count: 3 }) } }] + case 'merge': + // 合并查询：两个库各查一次、在库外按门店合并，带两道警告（键类型不一致、行数放大） + return [{ id: 'merge', question: '按门店合并销售和到店人数', phase: 'done', status: '已完成', + steps: decodeRun(mergeRun(), { status: 'succeeded' }) }] case 'schema': { const ev = mixedRun().slice(0, 10) return [{ id: 'schema', question: '看看有哪些人事表', phase: 'done', status: '已完成', diff --git a/frontend/src/index.css b/frontend/src/index.css index e81e5bc5..f1602470 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -72,6 +72,7 @@ --nt-supervisor: #bcadff; --nt-tool: #10c4c9; --nt-code: #57e3df; + --nt-merge: #96efe6; --nt-branch: #e389ca; --nt-loop: #ffaddd; --nt-subgraph: #dba9c8; @@ -130,6 +131,7 @@ --nt-supervisor: #63519e; --nt-tool: #026264; --nt-code: #057c79; + --nt-merge: #0a4d5a; --nt-branch: #7e376c; --nt-loop: #9b4c7d; --nt-subgraph: #7a516c; @@ -186,6 +188,7 @@ --nt-supervisor: #63519e; --nt-tool: #026264; --nt-code: #057c79; + --nt-merge: #0a4d5a; --nt-branch: #7e376c; --nt-loop: #9b4c7d; --nt-subgraph: #7a516c; @@ -393,6 +396,7 @@ .nt-supervisor { --nt: var(--nt-supervisor); } .nt-tool { --nt: var(--nt-tool); } .nt-code { --nt: var(--nt-code); } +.nt-merge { --nt: var(--nt-merge); } .nt-branch { --nt: var(--nt-branch); } .nt-loop { --nt: var(--nt-loop); } .nt-human { --nt: var(--nt-human); } diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index c9df87d7..e34d039e 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -25,6 +25,10 @@ * - 报错、修复选项、说明里写界面上的中文标签（「单元格引用」「报告来自」），不在句子里裸写键名和 * 枚举值；配置界面里字段自身旁边可以附键名。 * - 导航里的历史页叫「记录」。 + * - 「合并查询」是节点名，也指它做的事：把几次查询的结果在库外按键合并成一张表。不写联表、联邦查询、join。 + * 合并时两边对齐的列（日期、门店）叫「合并键」；合并 SQL 里代表某个输入的表名叫「别名」；被合并的那几次 + * 查询叫「输入」。合并结果里的一格能指回输入的哪一格时叫「逐格来历」，指不回时只有「表级来历」（合并了 + * 哪几个输入、用的哪条合并 SQL）。 * * 语体： * - 书面、简洁、中性。用「无法 / 未能 / 失败 / 请」，不用「取不到 / 连不上 / 写坏了」这类口语补语； @@ -120,6 +124,7 @@ export const NODE_TYPE_LABEL: Record = { supervisor: '多 Agent 协作', tool: '调用工具', code: '沙箱代码', + merge: '合并查询', branch: '条件分支', loop: '循环', subgraph: '子工作流', @@ -420,6 +425,41 @@ export const EVIDENCE_TEXT = { }, } as const +/** + * 合并查询（merge 节点）在检查器、运行详情、证据面板和推断来源里的说法。用到的术语（输入、别名、合并键、 + * 逐格来历、表级来历）见文件开头。行号对人一律从 1 数，和证据面板查询表格最前面那一列「#」一致 + */ +export const MERGE_TEXT = { + // ---- 检查器 ---- + addInput: '添加输入', + alias: '别名（表名）', + node: '查询节点', + pick: '选择上游的查询节点', + removeInput: (alias: string) => `删除输入 ${alias || '（未命名）'}`, + noUpstream: '上游没有查询节点。请先在本节点之前连接选择了数据库查询工具的「调用工具」节点，或另一个「合并查询」节点', + missingNode: (id: string) => `节点「${id}」不在上游或已删除，请重新选择`, + inputsHint: '别名就是合并 SQL 中的表名。各输入请先在自己的库中聚合到相同的合并键和粒度（例如日期 + 门店），任何一个输入被截断，合并都会失败', + // ---- 运行详情 ---- + stepTitle: (aliases: string[]) => (aliases.length ? `合并 ${aliases.join('、')} 的查询结果` : '合并查询结果'), + sql: '合并 SQL', + copySql: '复制合并 SQL', + inputs: (n: number) => `合并自 ${formatNumber(n)} 个输入`, + rows: (n: number) => `${formatNumber(n)} 行`, + source: (name: string) => `数据源 ${name}`, + warnings: (n: number) => `${formatNumber(n)} 条警告，详见本节点下方`, + // ---- 证据面板 ---- + mergedInto: (alias: string) => `合并查询 ${alias} 的输入`, + traced: '被引用的格来自', + cellAt: (alias: string, row: number, column: string) => `${alias} 第 ${formatNumber(row + 1)} 行「${column}」`, + noLineage: '没有逐格来历，只有表级来历：见上方的输入和合并 SQL', + unsealed: '不在封存范围内', + notInCatalog: '不在本报告的证据目录中', + executionWarnings: '执行时的警告', + // ---- 推断来源 ---- + hop: (alias: string, to: string) => `经合并查询 ${alias} 追到 ${to}`, + hopMissing: (alias: string) => `合并查询 ${alias} 中的这一格没有逐格来历`, +} as const + /** 结论句按裁判结论分成的几堆（lib/evidence 的 claimCounts 数出来的） */ export interface ClaimTallyCounts { /** 结论句总数：模型判为「不是结论句」的不算 */ diff --git a/frontend/src/run/AssistantStream.tsx b/frontend/src/run/AssistantStream.tsx index f2a14620..cb6aa5f3 100644 --- a/frontend/src/run/AssistantStream.tsx +++ b/frontend/src/run/AssistantStream.tsx @@ -15,7 +15,7 @@ import { humanizeError } from '../lib/errors' import { explainRunError } from '../lib/explain' import { formatClock, formatDuration, formatNumber, NONE, shortId } from '../lib/format' import { statusLabel } from '../lib/status' -import { EVIDENCE_TEXT, JUDGE_TEXT, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms' +import { EVIDENCE_TEXT, JUDGE_TEXT, MERGE_TEXT, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms' import { ISSUANCE_HINT, childrenByExec, compactSteps, parseQueryResult, progressOf, spread, teamVerdictOf, type Exec, type FixKind, type ResultTable as Table, type Step, type StepKind, type TeamMemberEx, @@ -723,7 +723,10 @@ function findFirst(steps: Step[], pred: (s: Step) => boolean): Step | undefined function countKind(steps: Step[], kind: StepKind): number { let n = 0 - for (const s of steps) n += (s.kind === kind ? 1 : 0) + (s.children ? countKind(s.children, kind) : 0) + // 合并查询那一行也是 kind query（同样的展开区和下钻），但它没有查数据库：「N 次查询」和落库的查询次数对得上 + for (const s of steps) { + n += (s.kind === kind && !s.merge ? 1 : 0) + (s.children ? countKind(s.children, kind) : 0) + } return n } @@ -1471,14 +1474,32 @@ function StepDetail({ step, table, explained }: { )} )} + {step.merge && step.merge.inputs.length > 0 && ( + // 合并查询：先说合并的是哪几个输入（合并 SQL 里的表名 → 上游节点、行数、数据源），再看合并 SQL 和结果 +
+
{MERGE_TEXT.inputs(step.merge.inputs.length)}
+
    + {step.merge.inputs.map((i) => ( +
  • + {i.alias} + {i.label || i.nodeId || '—'} + + {[i.rows != null ? MERGE_TEXT.rows(i.rows) : '', i.source ? MERGE_TEXT.source(i.source) : ''] + .filter(Boolean).join(' · ')} + +
    • + ))} +
+
+ )} {detail && (
{step.kind === 'query' && (
- SQL + {step.merge ? MERGE_TEXT.sql : 'SQL'} {step.source && 数据源 {step.source}} - step.detail ?? ''} /> + step.detail ?? ''} />
)} 
diff --git a/frontend/src/run/EvidencePanel.tsx b/frontend/src/run/EvidencePanel.tsx
index d651c78b..a7b88294 100644
--- a/frontend/src/run/EvidencePanel.tsx
+++ b/frontend/src/run/EvidencePanel.tsx
@@ -18,12 +18,12 @@ import {
 } from '../lib/evidence'
 import { humanizeError } from '../lib/errors'
 import { formatDateTime, formatNumber, NONE, shortId } from '../lib/format'
-import { EVIDENCE_TEXT, JUDGE_TEXT, nodeTypeLabel } from '../lib/terms'
+import { EVIDENCE_TEXT, JUDGE_TEXT, MERGE_TEXT, NODE_TYPE_LABEL, nodeTypeLabel } from '../lib/terms'
 import { useCatalog } from '../store/catalog'
 import { askKey, segmentKey, useEvidence, useExplore, useVerdicts } from '../store/evidence'
 import type {
-  EvidenceBlock, EvidenceDocData, EvidenceInput, EvidenceOnDemand, EvidenceQuoteSource, EvidenceSeal, EvidenceSegment,
-  EvidenceStep, EvidenceUnit, EvidenceVerdict, EvidenceViolation,
+  EvidenceBlock, EvidenceDocData, EvidenceInput, EvidenceMerge, EvidenceOnDemand, EvidenceQuoteSource, EvidenceSeal,
+  EvidenceSegment, EvidenceStep, EvidenceUnit, EvidenceVerdict, EvidenceViolation,
 } from '../types'
 import { ArtifactViewer, ResultTable } from './AssistantStream'
 import { useEvidenceHost } from './evidenceHost'
@@ -1346,16 +1346,39 @@ function QueryPart({ id, step, seal, masked }: { id: string; step: EvidenceStep;
               className="min-w-0 flex-1 truncate rounded-sm text-2xs font-medium text-faint focus:outline-none focus-visible:outline focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-offset-1 focus-visible:outline-[color:var(--accent)]">
           {EVIDENCE_TEXT.query(alias)}
           {step.tool && {step.tool}}
+          {step.merge && {NODE_TYPE_LABEL.merge}}
+          {step.merged_into && (
+            
+              · {MERGE_TEXT.mergedInto(step.merged_into)}
+            
+          )}
         
         {host.onNode && }
-        {step.sql &&  step.sql ?? ''} />}
+        {step.sql && (
+           step.sql ?? ''} />
+        )}
+ {step.merge && } + {step.sql && step.merge &&
{MERGE_TEXT.sql}
} {step.sql && ( 
           {step.sql}
)} + {!!step.merge?.warnings.length && ( +
+
{MERGE_TEXT.executionWarnings}
+
    + {step.merge.warnings.map((w, k) => ( +
  • + + {w.message} +
    • + ))} +
+
+ )} {bad.length > 0 && ( (k === 'hash' ? 'query-hash' : k)).join(' ')} items={bad.map((k) => (k === 'hash' ? EVIDENCE_TEXT.queryHash : EVIDENCE_TEXT.integrity[k]))} /> @@ -1371,6 +1394,7 @@ function QueryPart({ id, step, seal, masked }: { id: string; step: EvidenceStep; // 没有行：快照不在封存范围里、哈希对不上、取不回来。接口说了原因就照原话

{step.note || EVIDENCE_TEXT.queryMissing}

)} + {!!step.merge?.traced.length && }
{win.windowed && win.total != null && ( @@ -1405,6 +1429,59 @@ function QueryPart({ id, step, seal, masked }: { id: string; step: EvidenceStep; /** 窗口表格最前面那一列：快照里的行号，从 1 数 */ const ROW_NO = '#' +/** + * 合并查询的表级来历：合并了哪几个输入（合并 SQL 里的别名 → 报告目录里的编号、节点、行数、数据源）。 + * 输入不在这份报告的证据目录里时照实说：它的那次查询点不开 + */ +function MergeInputs({ merge }: { merge: EvidenceMerge }) { + const host = useEvidenceHost() + return ( +
+
{MERGE_TEXT.inputs(merge.inputs.length)}
+
    + {merge.inputs.map((i) => ( +
  • + {i.alias} + {i.query + ? {EVIDENCE_TEXT.query(i.query)} + : {MERGE_TEXT.notInCatalog}} + {i.label || i.node_id || NONE} + + {[typeof i.rows === 'number' ? MERGE_TEXT.rows(i.rows) : '', i.source ? MERGE_TEXT.source(i.source) : ''] + .filter(Boolean).join(' · ')} + + {host.onNode && i.node_id && } +
    • + ))} +
+
+ ) +} + +/** + * 被引用的格追到了哪个输入的哪一格（逐格来历）。追不到的照实说只有表级来历：宁可不指，也不指错。 + * 追到的那一格在下面那个输入的查询步骤里高亮 + */ +function MergeTraces({ alias, merge }: { alias: string; merge: EvidenceMerge }) { + return ( +
+
{MERGE_TEXT.traced}
+
    + {merge.traced.map((t, k) => ( +
  • + {MERGE_TEXT.cellAt(alias, t.cell[0], t.cell[1])} + ← + {t.input && t.row != null && t.column + ? {MERGE_TEXT.cellAt(t.query ?? t.input, t.row, t.column)} + : {MERGE_TEXT.noLineage}} +
    • + ))} +
+
+ ) +} + /** 接口没给查询步骤时：文档目录里记着的那次查询（编号、工具、行数），并照实说行没取到 */ function QueryFallback({ entry, alias, where, pending, failed }: { entry?: Record; alias?: string; where: string; pending: boolean; failed: string diff --git a/frontend/src/run/EvidenceProvenance.tsx b/frontend/src/run/EvidenceProvenance.tsx index b9de78e0..bbe49a1b 100644 --- a/frontend/src/run/EvidenceProvenance.tsx +++ b/frontend/src/run/EvidenceProvenance.tsx @@ -5,12 +5,12 @@ import clsx from 'clsx' import { ApiError, api } from '../api/client' import { ErrorState, Modal, Skeleton } from '../components/ui' import { NONE, formatDateTime } from '../lib/format' -import { EVIDENCE_TEXT, RAW_STATE_LABEL, VERSIONS_TEXT } from '../lib/terms' +import { EVIDENCE_TEXT, MERGE_TEXT, RAW_STATE_LABEL, VERSIONS_TEXT } from '../lib/terms' import { ManifestBody } from '../pages/import/ManifestBody' import { segmentKey, useEvidence } from '../store/evidence' import type { - EvidenceProvenance as Provenance, ManifestOut, ProvenanceCellSource, ProvenanceCheck, ProvenanceFromCell, ProvenancePart, - ProvenanceVersion, + EvidenceProvenance as Provenance, ManifestOut, ProvenanceCellSource, ProvenanceCheck, ProvenanceFromCell, ProvenanceMergeHop, + ProvenancePart, ProvenanceVersion, } from '../types' import { CopyChip } from './Markdown' @@ -106,7 +106,9 @@ function planOf(d: Provenance): Plan { const alert = d.alert?.code if (d.status === 'none') { if (!alert && (code === 'legacy_doc' || code === 'not_upload' || code === 'not_cell')) return NOTHING - if (!alert && (code === 'not_sealed' || code === 'simple_upload')) return { version: false, source: 'reason', checks: false } + if (!alert && (code === 'not_sealed' || code === 'simple_upload' || code === 'merge_no_lineage')) { + return { version: false, source: 'reason', checks: false } + } if (alert === 'manifest_unreadable' || alert === 'chain_mismatch') return { version: false, source: 'alert', checks: false } } else if (d.status === 'table_only') { if (alert === 'db_tampered' || alert === 'chain_mismatch') return { version: true, source: 'alert', checks: false } @@ -126,6 +128,7 @@ function Answer({ d }: { d: Provenance }) { <> {version && } + {Array.isArray(d.merge) && d.merge.length > 0 && } {plan.source === 'alert' && d.alert && ( // 标红：样式同 IntegrityList（失败色的框），放在这一节最前面；有它时不再画 reason.text

+ {hops.map((h, k) => ( +

  • + {h.input && h.row != null && h.column + ? MERGE_TEXT.hop(h.alias, MERGE_TEXT.cellAt(h.query ?? h.input, h.row, h.column)) + : MERGE_TEXT.hopMissing(h.alias)} +
    • + ))} + + ) +} + /** 「推断，不属于已封存的证据」：推断的来源、相关核对两节的标题旁都挂 */ function Badge() { return ( diff --git a/frontend/src/run/__tests__/evidence-provenance.json b/frontend/src/run/__tests__/evidence-provenance.json index 6175aed9..c8f1325f 100644 --- a/frontend/src/run/__tests__/evidence-provenance.json +++ b/frontend/src/run/__tests__/evidence-provenance.json @@ -6639,7 +6639,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s8": { "schema": "agentlab.provenance/1", @@ -6793,7 +6794,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s12": { "schema": "agentlab.provenance/1", @@ -6952,7 +6954,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s16": { "schema": "agentlab.provenance/1", @@ -7018,7 +7021,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s20": { "schema": "agentlab.provenance/1", @@ -7084,7 +7088,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s24": { "schema": "agentlab.provenance/1", @@ -7150,7 +7155,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s28": { "schema": "agentlab.provenance/1", @@ -7216,7 +7222,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s56": { "schema": "agentlab.provenance/1", @@ -7282,7 +7289,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s60": { "schema": "agentlab.provenance/1", @@ -7466,7 +7474,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s64": { "schema": "agentlab.provenance/1", @@ -7642,7 +7651,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s68": { "schema": "agentlab.provenance/1", @@ -7877,7 +7887,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s72": { "schema": "agentlab.provenance/1", @@ -8047,7 +8058,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s76": { "schema": "agentlab.provenance/1", @@ -8217,7 +8229,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s84": { "schema": "agentlab.provenance/1", @@ -8357,7 +8370,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s88": { "schema": "agentlab.provenance/1", @@ -8447,7 +8461,8 @@ "ok": true } }, - "checks": [] + "checks": [], + "merge": [] }, "s92": { "schema": "agentlab.provenance/1", @@ -8472,7 +8487,8 @@ "sealed": true, "version": null, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s52": { "schema": "agentlab.provenance/1", @@ -8626,7 +8642,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s32": { "schema": "agentlab.provenance/1", @@ -8695,7 +8712,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s36": { "schema": "agentlab.provenance/1", @@ -8764,7 +8782,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s48": { "schema": "agentlab.provenance/1", @@ -8830,7 +8849,8 @@ ] }, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s44": { "schema": "agentlab.provenance/1", @@ -8858,7 +8878,8 @@ "sealed": true, "version": null, "cell_source": null, - "checks": [] + "checks": [], + "merge": [] }, "s40": { "schema": "agentlab.provenance/1", @@ -9016,7 +9037,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] }, "s80": { "schema": "agentlab.provenance/1", @@ -9190,7 +9212,8 @@ "acceptance": null, "detail": null } - ] + ], + "merge": [] } }, "errors": { diff --git a/frontend/src/run/__tests__/synthetic.ts b/frontend/src/run/__tests__/synthetic.ts index b3231f76..ed777ba8 100644 --- a/frontend/src/run/__tests__/synthetic.ts +++ b/frontend/src/run/__tests__/synthetic.ts @@ -335,6 +335,45 @@ export function markupRun(): RunEvent[] { return out } +/** + * 合并查询：门店库、会员库各查一次，在库外按门店合并（engine/nodes/merge.py 的 merge.end 和随后的警告 log）。 + * 门店编号一边是文本、一边是数，又只按门店没按日期合并：两道警告都发 + */ +export const MERGE_SQL = 'SELECT s.门店, s.订单数, v.到店人数 FROM s JOIN v ON s.门店 = v.门店' +export function mergeRun(): RunEvent[] { + const { out, ev } = builder('syn-merge') + const query = (node: string, label: string, source: string, sql: string, columns: string[], rows: unknown[][]) => { + ev('node.started', node, { node_type: 'tool', label }) + ev('tool.start', node, { tool: `db_query__${source}`, args: { sql } }) + ev('tool.end', node, { tool: `db_query__${source}`, duration_ms: 40, artifact: `ts-${node}`, query_artifact: `qs-${node}`, + preview: JSON.stringify({ columns, rows, row_count: rows.length, truncated: false, sql, source }) }, 0.04) + ev('node.finished', node, { duration_ms: 45, attempt: 1 }) + } + ev('run.started', null, { nodes: 4, resumed: false }) + query('q_sales', '门店销售', 'stores', 'SELECT order_date AS 日期, substr(store_id, 2) AS 门店, COUNT(*) AS 订单数 FROM orders GROUP BY 1, 2', + ['日期', '门店', '订单数'], [['2026-05-01', '01', 3], ['2026-05-01', '02', 2], ['2026-05-02', '01', 2], ['2026-05-02', '02', 1]]) + query('q_visits', '到店人数', 'members', 'SELECT visit_date AS 日期, CAST(substr(store_code, 2) AS INTEGER) AS 门店, COUNT(*) AS 到店人数 FROM visits GROUP BY 1, 2', + ['日期', '门店', '到店人数'], [['2026-05-01', 1, 6], ['2026-05-01', 2, 4], ['2026-05-02', 1, 5], ['2026-05-02', 2, 4]]) + ev('node.started', 'merge', { node_type: 'merge', label: '按门店合并' }) + ev('merge.end', 'merge', { + inputs: [ + { alias: 's', node_id: 'q_sales', label: '门店销售', rows: 4, source: 'stores', artifact: 'qs-q_sales' }, + { alias: 'v', node_id: 'q_visits', label: '到店人数', rows: 4, source: 'members', artifact: 'qs-q_visits' }, + ], + sql: MERGE_SQL, rows: 8, columns: ['门店', '订单数', '到店人数'], + preview_rows: [['01', 3, 6], ['01', 3, 5], ['01', 2, 6], ['01', 2, 5], ['02', 2, 4]], + truncated: false, duration_ms: 12, query_artifact: 'mq-merge', lineage: true, + warnings: [{ code: 'key_type_mismatch', message: '键类型' }, { code: 'rows_grew', message: '行数' }], + }, 0.02) + ev('log', 'merge', { level: 'warn', code: 'merge_key_type', + message: "合并键类型不一致：s.门店 是文本（例如 '01'），v.门店 是数值（例如 1）。SQLite 比较时会做隐式转换：文本形式的编号（如 '001'、'01'）都会等于数值 1，也可能完全匹配不上。请在源查询中统一类型，或在合并 SQL 中用 CAST 明确转换" }) + ev('log', 'merge', { level: 'warn', code: 'merge_rows_grew', + message: '合并结果有 8 行，多于行数最多的输入「s」（4 行）：合并键可能不唯一，同一行被重复匹配。「s」中按（门店）有重复的键，例如 01 出现 2 次等 2 组；「v」中按（门店）有重复的键，例如 1 出现 2 次等 2 组。请检查合并条件是否覆盖了全部键（例如同时按日期和门店），或先在源库里聚合到相同粒度' }) + ev('node.finished', 'merge', { duration_ms: 15, attempt: 1 }) + ev('run.finished', null, { output: {}, timing: { wall_ms: 400, active_ms: 400, wait_ms: 0 } }) + return out +} + /** 结构化校验：原文里没有数，修复两次都编出了 total_count=0，两次都作废，判失败 */ export function repairRun(): RunEvent[] { const { out, ev } = builder('syn-repair') diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index 6b0dde52..5348c930 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -1,7 +1,7 @@ import type { RunEvent, TeamMember, TeamRound, TeamRun, ToolChange } from '../types' import type { NodeState, RunPhase } from './trace' import { formatDuration, formatNumber } from '../lib/format' -import { JUDGE_TEXT, TYPE_LABEL, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms' +import { JUDGE_TEXT, MERGE_TEXT, TYPE_LABEL, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms' import { claimCountsOf, isJudged, statsTally } from '../lib/evidence' // 泳道数据画布也要用（supervisor 节点要展开成协作矩阵），所以类型放在 @@ -145,6 +145,25 @@ export interface Step { limitS?: number /** 出具那一行的档位（formal / degraded / withheld） */ tier?: string + /** 合并查询那一行（merge.end）：合并了哪几个输入、几条警告。SQL 在 detail，预览在 result */ + merge?: MergeStep +} + +/** 合并查询的一个输入：合并 SQL 里的别名、上游节点、行数、数据源 */ +export interface MergeStepInput { + alias: string + nodeId?: string + label?: string + rows?: number + source?: string +} + +export interface MergeStep { + inputs: MergeStepInput[] + /** 执行时的警告条数。警告的原文由同一节点紧接着的 log（merge_key_type…）各占一行，不在这里重复 */ + warnings: number + rows?: number + truncated: boolean } // 这两个是流式增量，后端根本不落库（_EPHEMERAL）。单次运行的 delta 量级会 @@ -766,6 +785,38 @@ function explainLog(code: string | undefined, message: string): Pick x[1]) + return { + title: keys.length === 2 ? `合并键类型不一致：${keys[0]} 与 ${keys[1]}` : '合并键类型不一致', + sub: m?.[1], + next: '两边类型不同时，SQLite 按隐式转换比较，可能错配或完全匹配不上。请在源查询中统一类型，或在合并 SQL 中用 CAST 明确转换', + fix: 'canvas', + } + } + case 'merge_rows_grew': { + // 「合并结果有 8 行，多于行数最多的输入「s」（4 行）：合并键可能不唯一，同一行被重复匹配。…」 + const head = message.match(/^合并结果有\s*([\d,]+)\s*行，多于行数最多的输入「(.+?)」（([\d,]+)\s*行）/) + const dup = message.match(/重复匹配。(.+?)。请检查/)?.[1] + return { + title: '合并结果行数多于输入，合并键可能不唯一', + sub: [head ? `合并结果 ${head[1]} 行，行数最多的输入「${head[2]}」${head[3]} 行` : '', dup ?? ''] + .filter(Boolean).join('；') || undefined, + next: '请检查合并条件是否覆盖了全部合并键（例如同时按日期和门店），或先在源库里聚合到相同粒度', + fix: 'canvas', + } + } + case 'merge_truncated': { + const rows = message.match(/只保留了前\s*([\d,]+)\s*行/)?.[1] + return { + title: `合并结果超过上限，${rows ? `只保留了前 ${rows} 行` : '已截断'}`, + next: '下游拿到的不是完整结果。请在合并 SQL 中聚合或加条件缩小范围', + fix: 'canvas', + } + } case 'judge_failed': { // 「结论句裁判没跑完：有 2 句结论没裁判：裁判调用失败（…）」 return { @@ -2085,6 +2136,38 @@ export function decodeRun(events: RunEvent[], final?: RunFinal, opts?: DecodeOpt break } + case 'merge.end': { + // 合并查询：几次查询的结果在库外按键合并。和查询同一类（kind query）：detail 是合并 SQL，result 是预览， + // 工件是合并结果的查询快照。输入列在 merge 里，展开区单列一块；警告的原文由紧跟着的 log 各占一行 + const inputs: MergeStepInput[] = (Array.isArray(d.inputs) ? d.inputs : []) + .filter((i: any) => i && typeof i === 'object' && typeof i.alias === 'string') + .map((i: any) => ({ + alias: String(i.alias), + ...(typeof i.node_id === 'string' ? { nodeId: i.node_id } : {}), + ...(typeof i.label === 'string' && i.label ? { label: i.label } : {}), + ...(num(i.rows) != null ? { rows: num(i.rows) } : {}), + ...(typeof i.source === 'string' && i.source ? { source: i.source } : {}), + })) + const warnings = Array.isArray(d.warnings) ? d.warnings.filter((w: any) => w && w.message).length : 0 + const rows = num(d.rows) + const ms = num(d.duration_ms) + const preview = Array.isArray(d.preview_rows) ? d.preview_rows : [] + push({ + id: `mg-${seq}`, seq, kind: 'query', nodeId, status: 'done', code: 'merge', + ...(warnings ? { level: 'warn' as const, sub: MERGE_TEXT.warnings(warnings) } : {}), + title: MERGE_TEXT.stepTitle(inputs.map((i) => i.alias)), + detail: typeof d.sql === 'string' && d.sql ? d.sql : undefined, + result: Array.isArray(d.columns) + ? JSON.stringify({ columns: d.columns, rows: preview, truncated: rows != null && rows > preview.length }) + : undefined, + meta: [rows != null ? MERGE_TEXT.rows(rows) : '', dur(ms) ?? ''].filter(Boolean).join(' · ') || undefined, + ms, + ...(typeof d.query_artifact === 'string' && d.query_artifact ? { artifact: d.query_artifact } : {}), + merge: { inputs, warnings, ...(rows != null ? { rows } : {}), truncated: !!d.truncated }, + }, nodeId) + break + } + case 'retrieve.end': { // 知识检索。以前只发一条 info 日志，于是"这句结论依据的是哪份文档的 // 哪一段"在轨迹里查不到——而 SQL 取数那条路早就能下钻到工件 diff --git a/frontend/src/types.ts b/frontend/src/types.ts index c250a8bd..286f26df 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -1,5 +1,5 @@ export type NodeType = - | 'input' | 'output' | 'llm' | 'agent' | 'supervisor' | 'tool' | 'code' + | 'input' | 'output' | 'llm' | 'agent' | 'supervisor' | 'tool' | 'code' | 'merge' | 'branch' | 'loop' | 'subgraph' | 'memory' | 'retrieve' | 'transform' | 'human' | 'validate' | 'metrics' | 'report' @@ -2065,6 +2065,40 @@ export interface EvidenceCaliberUpgrade { * run_input（报告直接引用的运行输入）、query（查询快照：被引用的行加前后各 2 行）。 * input 步骤同时带着 EvidenceInput 的字段（via、status、model_value、ref、locator、artifact） */ +/** 合并查询的一个输入（证据面板的 merge.inputs） */ +export interface EvidenceMergeInput { + /** 合并 SQL 里的表名 */ + alias: string + node_id?: string | null + /** 节点在画布上的名字 */ + label?: string | null + /** 这个输入在报告目录里的编号（Q1）；不在这份报告的目录里时为空 */ + query?: string | null + rows?: number | null + /** 数据源名；输入本身是合并结果时是「合并查询」 */ + source?: string | null + artifact?: string | null + sealed?: boolean +} + +/** 被引用的一格追到了哪个输入的哪一格；追不到时 input 为空，note 说明只有表级来历 */ +export interface EvidenceMergeTrace { + /** 合并结果里的 [行, 列]，行号从 0 数 */ + cell: [number, string] + input: string | null + query: string | null + row: number | null + column: string | null + note?: string +} + +export interface EvidenceMerge { + sql?: string | null + inputs: EvidenceMergeInput[] + warnings: { code?: string | null; message: string }[] + traced: EvidenceMergeTrace[] +} + export interface EvidenceStep extends Omit { step: string metric?: string @@ -2133,6 +2167,10 @@ export interface EvidenceStep extends Omit { * 查询快照来自上传的表格时才有，且恒为 true；其余一个键都不加。前端只在 seg.cite?.kind === 'cell' 且它为 true 时请求 */ provenance?: boolean + /** 合并查询的结果（快照的 source 是「合并查询」）：合并了哪几个输入、合并 SQL、执行时的警告、被引用的格追到哪 */ + merge?: EvidenceMerge + /** 这一步是哪次合并查询的输入（合并结果在目录里的编号）：排在那个合并步骤后面，高亮的是追到的格 */ + merged_into?: string // ---- entity：表或字段（三期）---- /** entity：table / column */ kind?: string @@ -2268,7 +2306,7 @@ export type ProvenanceReasonCode = | 'legacy_doc' | 'not_cell' | 'not_sealed' | 'not_upload' | 'simple_upload' | 'manifest_unreadable' | 'chain_mismatch' | 'expression' | 'alias' | 'multi_table' | 'unparsed' | 'no_pk' | 'pk_missing' | 'masked' | 'null_value' | 'null_pk' | 'snapshot_gone' | 'db_tampered' | 'recheck_missing' | 'recheck_multiple' - | 'recheck_mismatch' | 'no_lineage' + | 'recheck_mismatch' | 'no_lineage' | 'merge_no_lineage' /** 标红的提示：出现时 reason.code 与它相同，界面只画提示 */ export type ProvenanceAlertCode = 'db_tampered' | 'chain_mismatch' | 'manifest_unreadable' @@ -2476,4 +2514,19 @@ export interface EvidenceProvenance { version: ProvenanceVersion | null cell_source: ProvenanceCellSource | null checks: ProvenanceCheck[] + /** 被引用的格在合并查询的结果里时，经过的每一次合并；别的查询为空 */ + merge: ProvenanceMergeHop[] +} + +/** 经过的一次合并查询：合并结果里被引用的格追到了哪个输入的哪一格；追不到的那一跳后四项为 null */ +export interface ProvenanceMergeHop { + /** 合并结果在目录里的编号（Q3） */ + alias: string + node_id: string | null + /** 追到的输入别名（合并 SQL 里的表名） */ + input: string | null + /** 那个输入在目录里的编号（Q1） */ + query: string | null + row: number | null + column: string | null } diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index 056f54d2..e8f4e9ee 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -112,6 +112,33 @@ await section('知识检索', async () => { check('未知事件的类型名不上标题和副标题', !unknown.some((x) => x.title.includes('brand.new') || x.sub?.includes('brand.new'))) }) +await section('合并查询', async () => { + // merge.end 是合并查询节点做完的那一条：哪几个输入、合并 SQL、几行。和查询同一类（kind query）， + // 合并 SQL 进详情、预览进结果、工件可下钻；警告的原文由紧跟着的 log 各占一行，这一行只说有几条 + const steps = mod.decodeRun(synthetic.mergeRun(), { status: 'succeeded' }) + const all = flatten(steps) + const m = all.find((s) => s.code === 'merge') + check('合并查询是一行查询步骤，挂在合并节点下', !!m && m.kind === 'query' && m.nodeId === 'merge' + && steps.some((s) => s.kind === 'node' && s.nodeId === 'merge' && s.children?.includes(m)), m?.kind) + check('标题说合并的是哪几个输入', m?.title === '合并 s、v 的查询结果', m?.title) + check('合并 SQL 留在详情里', m?.detail === synthetic.MERGE_SQL) + check('预览按查询结果的形状给出，能画成表格', !!mod.parseQueryResult(m?.result ?? '')?.columns?.includes('到店人数')) + check('输入、行数、数据源都在', m?.merge?.inputs?.length === 2 && m.merge.inputs[1].source === 'members' + && m.merge.inputs[0].label === '门店销售' && m.merge.rows === 8) + check('行数和耗时写在行尾', !!m?.meta?.includes('8 行'), m?.meta) + check('工件可下钻到合并结果的快照', m?.artifact === 'mq-merge') + check('有警告时这一行标成警告、说有几条', m?.level === 'warn' && !!m.sub?.includes('2 条警告'), m?.sub) + const key = all.find((s) => s.code === 'merge_key_type') + check('键类型不一致说成人话：标题点名两列', !!key?.title.includes('s.门店') && key.title.includes('v.门店'), key?.title) + check('键类型不一致给出下一步', !!key?.next?.includes('CAST') && key.fix === 'canvas') + const grew = all.find((s) => s.code === 'merge_rows_grew') + check('行数放大说成人话，并带上行数和重复的键', !!grew?.title.includes('不唯一') && !!grew.sub?.includes('8 行') + && !!grew.sub?.includes('出现 2 次'), grew?.sub) + check('警告原文各占一行，不在合并那一行里重复', all.filter((s) => s.level === 'warn' && s.nodeId === 'merge').length === 3) + check('合并节点照常完成，两次源查询仍是普通查询步骤', all.filter((s) => s.kind === 'query' && s.code !== 'merge').length === 2 + && steps.find((s) => s.nodeId === 'merge')?.status === 'done') +}) + await section('长期记忆', async () => { // 写记忆以前只发 info 日志，而 info 在这一层是被丢弃的——系统往长期记忆里 // 存东西，界面上一点痕迹都没有。Copilot 会主动记之后，这条不可接受。 diff --git a/scripts/check-evidence.mjs b/scripts/check-evidence.mjs index f6ddf70f..131a1efb 100644 --- a/scripts/check-evidence.mjs +++ b/scripts/check-evidence.mjs @@ -2989,7 +2989,96 @@ if (SHOTS) { await page.keyboard.press('Escape') } } -await section('copy-r2', '按后端原文匹配的地方：后端现在的原文、整改前的原文都认得（文案整改第二轮）', async () => { +// --------------------------------------------------------------------------- +// 合并查询：片段接口的查询步骤带 merge（表级来历、逐格来历），后面接着追到的输入的查询步骤（merged_into）； +// 推断来源的答复带 merge（经过的每一次合并）。答复按 backend/app/api/evidence.py 的 _merge_view、_query_steps 的形状改 +// --------------------------------------------------------------------------- +const MERGE_SQL = 'SELECT o.order_id, o.region, o.amount, t.target FROM o JOIN t ON o.region = t.region ORDER BY o.order_id' +function asMerge(body, traced, { withInput = true } = {}) { + const q = body.chain.find((st) => st.step === 'query') + const merged = { + ...q, tool: undefined, source: '合并查询', sql: MERGE_SQL, + merge: { + sql: MERGE_SQL, + inputs: [ + { alias: 'o', node_id: 'fetch', label: '订单明细', query: 'Q1', rows: 12, source: 'shop', artifact: 'a'.repeat(64), sealed: true }, + { alias: 't', node_id: 'targets', label: '区域目标', query: null, rows: 4, source: 'plan', artifact: 'b'.repeat(64), sealed: true }, + ], + warnings: [{ code: 'rows_grew', message: '合并结果有 14 行，多于行数最多的输入「o」（12 行）：合并键可能不唯一，同一行被重复匹配' }], + traced, + }, + } + const input = { ...q, alias: 'Q1', merged_into: q.alias } + return { ...body, chain: body.chain.flatMap((st) => (st.step === 'query' ? (withInput ? [merged, input] : [merged]) : [st])) } +} + +await section('merge', '合并查询：表级来历（输入、合并 SQL、警告）、逐格来历、输入步骤标明是哪次合并的输入', async () => { + const traced = await probeQ((id, body) => (id === qseg('1,288') + ? asMerge(body, [{ cell: [5, 'amount'], input: 'o', query: 'Q1', row: 5, column: 'amount' }]) : body)) + await openQ(traced.page, '1,288') + if (SHOTS) { + for (const theme of ['dark', 'light']) { + await traced.page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + await traced.page.waitForTimeout(250) + await traced.page.screenshot({ path: `${SHOTS}/evidence-merge-${theme}.png`, fullPage: true }) + } + await traced.page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark')) + } + const p = traced.page.locator('[data-evidence-panel]') + const parts = p.locator('[data-ev-query]') + check('合并步骤和它追到的输入步骤各一块', await parts.count() === 2, String(await parts.count())) + const head = await parts.first().locator('[data-ev-query-head]').innerText().catch(() => '') + check('合并步骤的标题写「合并查询」，不写工具名', head.includes('Q3') && head.includes('合并查询') && !head.includes('db_query__'), head) + const inputs = p.locator('[data-ev-merge]') + check('表级来历：合并自 2 个输入', (await inputs.innerText().catch(() => '')).includes('合并自 2 个输入')) + const o = await p.locator('[data-ev-merge-input="o"]').innerText().catch(() => '') + check('……每个输入写别名、目录编号、节点名、行数、数据源', ['o', '查询 Q1', '订单明细', '12 行', '数据源 shop'].every((t) => o.includes(t)), o) + const t = await p.locator('[data-ev-merge-input="t"]').innerText().catch(() => '') + check('……不在本报告目录里的输入照实说', t.includes('不在本报告的证据目录中'), t) + check('合并 SQL 有标签、带复制', (await parts.first().innerText()).includes('合并 SQL') + && await parts.first().locator('button', { hasText: '复制合并 SQL' }).count() === 1) + const warn = await p.locator('[data-ev-merge-warnings]').innerText().catch(() => '') + check('执行时的警告写出来', warn.includes('执行时的警告') && warn.includes('不唯一'), warn) + const line = await p.locator('[data-ev-merge-trace="cell"]').innerText().catch(() => '') + check('逐格来历：被引用的格追到输入的哪一格（行号从 1 数）', line.includes('Q3 第 6 行「amount」') && line.includes('Q1 第 6 行「amount」'), line) + const into = await parts.nth(1).locator('[data-ev-merged-into]').innerText().catch(() => '') + check('输入步骤标明是哪次合并的输入，高亮追到的那一格', into.includes('合并查询 Q3 的输入') + && await parts.nth(1).locator('td[data-highlight="cell"]').count() === 1, into) + check('合并查询不是数据源：不说它「已不存在」', await p.locator('[data-ev-query-mask-note]').count() === 0) + await traced.ctx.close() + + const flat = await probeQ((id, body) => (id === qseg('1,288') ? asMerge(body, [{ + cell: [5, 'amount'], input: null, query: null, row: null, column: null, note: '没有逐格来历', + }], { withInput: false }) : body)) + await openQ(flat.page, '1,288') + const none = await flat.page.locator('[data-evidence-panel] [data-ev-merge-trace="none"]').innerText().catch(() => '') + check('追不到的格：写明没有逐格来历、只有表级来历，不接输入步骤', none.includes('没有逐格来历') + && none.includes('表级来历') && await flat.page.locator('[data-evidence-panel] [data-ev-query]').count() === 1, none) + await flat.ctx.close() + + const wideId = pseg('wide') + const hop = { alias: 'Q3', node_id: 'merge', input: 'f', query: 'Q1', row: 5, column: '全日客流' } + const lost = { alias: 'Q3', node_id: 'merge', input: null, query: null, row: null, column: null } + const noLineage = { + ...fxp.provenance[wideId], status: 'none', version: null, cell_source: null, checks: [], merge: [lost], + reason: { code: 'merge_no_lineage', detail: '', text: '这一格来自合并查询，无法确定它对应哪个输入的哪一格' }, + } + const hops = await open('/ui-harness.html?evidence=1', { + prov: (run, sid, reply) => (sid === wideId ? { json: { ...reply.json, merge: [hop] } } + : sid === pseg('long') ? { json: noLineage } : reply), + }) + await openP(hops.page, 'wide') + const said = await textOf(pv(hops.page, '[data-ev-prov-merge]')) + check('推断的来源：先说经过的合并、追到的那一格，下面的原表格子照常', said === '经合并查询 Q3 追到 Q1 第 6 行「全日客流」' + && await pv(hops.page, '[data-ev-prov-cell]').count() === 1, said) + await openP(hops.page, 'long') + check('追不到逐格来历：写原因和没追到的那一跳，不画数据版本和格子', (await textOf(pv(hops.page, '[data-ev-prov-reason="merge_no_lineage"]'))) + .includes('合并查询') && (await textOf(pv(hops.page, '[data-ev-prov-merge-hop="none"]'))).includes('没有逐格来历') + && await pv(hops.page, '[data-ev-prov-version], [data-ev-prov-cell]').count() === 0) + await hops.ctx.close() +}) + +await section('copy-r2','按后端原文匹配的地方：后端现在的原文、整改前的原文都认得（文案整改第二轮）', async () => { // 画布问题落到哪一栏（canvas/issues.ts）、检查器的分支标识过滤（Inspector 用 issues.ts 的 isCaseKeyIssue）、 // 证据面板的裁判触顶（lib/evidence.ts）都按后端原文判断。开发库里历史运行、旧版本存下的文字是整改前的， // 新的是整改后的：两种各喂一遍。「新」样本和 backend/app 现在的写法一致（先核对源码片段）， diff --git a/scripts/check-stream.mjs b/scripts/check-stream.mjs index 10075035..b7938867 100644 --- a/scripts/check-stream.mjs +++ b/scripts/check-stream.mjs @@ -694,6 +694,43 @@ await section('长表名、Copilot 结局、问数据的分段', async () => { await q.page.close() }) +await section('合并查询：输入、合并 SQL、结果预览，警告各占一行并给出下一步', async () => { + const { page, errors } = await open('syn=merge', { w: 1100, h: 900, name: 'merge' }) + const row = page.locator('[data-step-code="merge"]').first() + const line = (await row.innerText().catch(() => '')).replace(/\s+/g, ' ') + check('合并那一行说合并的是哪几个输入，行尾写行数', line.includes('合并 s、v 的查询结果') && line.includes('8 行'), line) + check('有警告：这一行标成警告、说有几条、指到下方', line.includes('2 条警告，详见本节点下方'), line) + await row.locator('button').first().click() + await page.waitForTimeout(200) + const merge = row.locator('[data-step-merge]') + const inputs = (await merge.innerText().catch(() => '')).replace(/\s+/g, ' ') + check('展开：合并自 2 个输入，每个写别名、节点名、行数、数据源', inputs.includes('合并自 2 个输入') + && ['s', '门店销售', 'v', '到店人数', '4 行', '数据源 stores', '数据源 members'].every((t) => inputs.includes(t)), inputs) + check('合并 SQL 有标签、带复制', (await row.innerText()).includes('合并 SQL') + && await row.getByRole('button', { name: '复制合并 SQL' }).count() === 1) + check('结果预览画成表格（列是合并结果的列）', (await row.locator('table th').allInnerTexts()).some((t) => t.includes('到店人数'))) + check('输入写在合并 SQL 前面：先看合并的是什么，再看怎么合并', await row.evaluate((el) => { + const m = el.querySelector('[data-step-merge]') + const pre = el.querySelector('pre') + return !!m && !!pre && !!(m.compareDocumentPosition(pre) & Node.DOCUMENT_POSITION_FOLLOWING) + })) + const key = page.locator('[data-step-code="merge_key_type"]').first() + check('键类型不一致：标题点名两列，下一步不用展开就看得到', (await key.innerText().catch(() => '')).includes('合并键类型不一致：s.门店 与 v.门店') + && (await key.locator('[data-step-next]').innerText().catch(() => '')).includes('CAST')) + const grew = page.locator('[data-step-code="merge_rows_grew"]').first() + check('行数放大：说合并键可能不唯一，下一步说怎么改', (await grew.innerText().catch(() => '')).includes('合并键可能不唯一') + && (await grew.locator('[data-step-next]').innerText().catch(() => '')).includes('聚合到相同粒度')) + check('没有运行时报错', errors.length === 0, errors.join(' | ')) + await page.close() + + const narrow = await open('syn=merge&dense=1', { w: 380, h: 900 }) + await narrow.page.locator('[data-step-code="merge"] button').first().click() + await narrow.page.waitForTimeout(200) + const overflow = await narrow.page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth) + check('窄栏展开合并那一行：输入清单、合并 SQL、预览都不撑出横向滚动', overflow <= 0, `${overflow}px`) + await narrow.page.close() +}) + await section('步骤行和画布联动', async () => { const { page } = await open('syn=mixed&link=1', { w: 1100, h: 800 }) const row = page.locator('[data-node-id="agent"]').first() diff --git a/scripts/check-studio.mjs b/scripts/check-studio.mjs index f99f3caa..8177f7a6 100644 --- a/scripts/check-studio.mjs +++ b/scripts/check-studio.mjs @@ -114,6 +114,23 @@ const EV_GRAPH = { edges: [edge('start', 'fetch'), edge('start', 'manual'), edge('fetch', 'write'), edge('manual', 'write'), edge('write', 'done')], } +// 合并查询：门店库、会员库各查一次，按日期 + 门店合并。上游另有一个模型调用节点，检查器的输入下拉里不该有它 +const MERGE_GRAPH = { + nodes: [ + node('start', 'input', 0, 200, '输入', { fields: [{ name: 'day', required: true }] }), + node('q_sales', 'tool', 300, 80, '门店销售', { tool: 'db_query__shop', args: { sql: 'SELECT order_date AS 日期, store_id AS 门店, COUNT(*) AS 订单数 FROM orders GROUP BY 1, 2' } }), + node('q_visits', 'tool', 300, 240, '到店人数', { tool: 'db_query__members', args: { sql: 'SELECT visit_date AS 日期, store_code AS 门店, COUNT(*) AS 到店人数 FROM visits GROUP BY 1, 2' } }), + node('think', 'llm', 300, 400, '先想一想', { prompt: '{{ input.day }}' }), + node('merge', 'merge', 640, 200, '按门店合并', { + inputs: { s: 'q_sales', v: 'q_visits' }, + sql: 'SELECT s.日期, s.门店, s.订单数, v.到店人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店', + }), + node('done', 'output', 980, 200, '成果', { fields: [{ name: 'answer', value: '{{ nodes.merge.rows }}' }] }), + ], + edges: [edge('start', 'q_sales'), edge('start', 'q_visits'), edge('start', 'think'), edge('q_sales', 'merge'), + edge('q_visits', 'merge'), edge('think', 'merge'), edge('merge', 'done')], +} + const wf = (id, extra) => ({ id, name: extra.name, description: extra.description ?? '检查脚本伪造的工作流', graph: extra.graph ?? GRAPH, tags: extra.tags ?? [], version: extra.version ?? 3, is_template: !!extra.is_template, @@ -128,6 +145,7 @@ const FAKES = { 'st-team': wf('st-team', { name: '__studio_check_team__', graph: TEAM_BOUND, version: 2, published_version: 2, status: 'published' }), 'st-lint': wf('st-lint', { name: '__studio_check_lint__', graph: TEAM_GRAPH, version: 1 }), 'st-ev': wf('st-ev', { name: '__studio_check_ev__', graph: EV_GRAPH, version: 2 }), + 'st-merge': wf('st-merge', { name: '__studio_check_merge__', graph: MERGE_GRAPH, version: 1 }), } const VERSIONS = [ { id: 'v3', version: 3, note: '', created_at: '2026-09-26T02:00:00Z' }, @@ -2353,6 +2371,69 @@ await section('证据路径：点开报告片段时画布高亮、节点名能 await race.ctx.close() }) +// ================================================================ 合并查询 +await section('合并查询：节点库认得、卡片摘要、检查器从上游查询节点里选输入、别名重名先不写回、定位落到那一行', async () => { + const { ctx, page, errors } = await open({ path: '/studio/st-merge' }) + const entry = page.locator('[aria-label="节点库"] button[data-node-type="merge"]') + check('节点库「执行」一类里有「合并查询」', await entry.count() === 1 && (await entry.innerText()).includes('合并查询')) + const card = page.locator('.react-flow__node[data-id="merge"]') + check('卡片摘要写合并的是哪几张表（别名）', (await card.innerText().catch(() => '')).includes('合并 s、v'), + (await card.innerText().catch(() => '')).replace(/\s+/g, ' ').slice(0, 80)) + await S(page, () => window.__studio.getState().select('merge')) + await page.waitForSelector('[data-inspector-sheet] [data-field="inputs"]', { timeout: 4000 }).catch(() => {}) + // STUDIO_SHOTS=<目录>：亮暗两套各截一张合并查询的检查器，供人眼复核 + if (process.env.STUDIO_SHOTS) { + for (const theme of ['dark', 'light']) { + await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + await page.waitForTimeout(250) + await page.screenshot({ path: `${process.env.STUDIO_SHOTS}/studio-merge-${theme}.png` }) + } + await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark')) + } + const box = page.locator('[data-inspector-sheet] [data-field="inputs"]') + const aliases = await box.locator('input.mono').evaluateAll((els) => els.map((e) => e.value)) + check('每个输入一行：别名 s、v，各配一个查询节点', aliases.join(',') === 's,v', aliases.join(',')) + const options = await box.locator('select').first().locator('option:not([disabled])').evaluateAll((els) => els.map((e) => e.value)) + check('下拉只列上游的查询节点（调用数据库查询工具的节点），不列模型调用', options.join(',') === 'q_sales,q_visits' + || options.join(',') === 'q_visits,q_sales', options.join(',')) + check('选中的节点下面写着它调用的工具', (await box.innerText()).includes('db_query__shop')) + check('合并 SQL 用代码编辑框', await page.locator('[data-inspector-sheet] [data-field="sql"] textarea, [data-inspector-sheet] [data-field="sql"] [contenteditable]').count() >= 1) + const config = () => S(page, () => JSON.stringify(window.__studio.getState().nodes.find((n) => n.id === 'merge')?.data.config.inputs)) + + await box.locator('input.mono').nth(1).fill('S') + await page.waitForTimeout(150) + check('别名改成和另一行只差大小写：行内说明重复，先不写回（两行都还在）', + (await box.innerText()).includes('重复') && await config() === JSON.stringify({ s: 'q_sales', v: 'q_visits' }), await config()) + await box.locator('input.mono').nth(1).fill('visits') + await page.waitForTimeout(150) + check('改成不重名的别名：写回，顺序不变', await config() === JSON.stringify({ s: 'q_sales', visits: 'q_visits' }), await config()) + await box.locator('input.mono').nth(1).fill('9v') + await page.waitForTimeout(150) + check('不合法的别名当场说明写法', (await box.innerText()).includes('只能用英文字母、数字和下划线')) + await box.locator('input.mono').nth(1).fill('v') + await box.getByRole('button', { name: '添加输入' }).click() + await page.waitForTimeout(150) + const added = JSON.parse(await config()) + check('添加输入：取一个上游查询节点，别名从节点 id 起、不重名', Object.keys(added).length === 3 + && Object.keys(added)[2] === 'q_sales', JSON.stringify(added)) + await box.getByRole('button', { name: '删除输入 q_sales' }).click() + await page.waitForTimeout(150) + check('删除输入：那一行和配置里的那一项一起去掉', await config() === JSON.stringify({ s: 'q_sales', v: 'q_visits' }), await config()) + + await S(page, () => window.__studio.getState().select(null)) + await page.waitForTimeout(200) + const r = await page.evaluate(async () => { + const m = await window.__appImport('/src/canvas/InspectorSheet.tsx') + m.revealField('merge', 'inputs.v', { focus: true }) + await new Promise((res) => setTimeout(res, 600)) + const a = document.activeElement + return { inRow: !!a?.closest('[data-field="inputs"] [data-sub="v"]'), value: (a && 'value' in a) ? a.value : null } + }).catch((e) => ({ error: e.message.split('\n')[0] })) + check('问题落在 inputs.<别名> 上：定位到那一行，光标落进别名', r.inRow && r.value === 'v', JSON.stringify(r)) + check('没有运行时报错', errors.length === 0, errors.slice(0, 2).join(' | ')) + await ctx.close() +}) + check('整个检查没有弹出原生 confirm / prompt', dialogs === 0, `${dialogs} 次`) check('检查没有写库（兜底拦下的写请求）', writes.length === 0, writes.slice(0, 4).join('、'))