diff --git a/backend/app/core/events.py b/backend/app/core/events.py index 81ed9c40..abef3e77 100644 --- a/backend/app/core/events.py +++ b/backend/app/core/events.py @@ -48,6 +48,9 @@ class EventType(StrEnum): # 记忆读写。以前发的是 info 日志,而解码器丢弃所有 info——于是往长期记忆里 # 写东西这件事在界面上是隐形的。让 Copilot 主动记之后,这条不可接受 MEMORY_END = "memory.end" + # 合并查询做完了:哪几个输入(别名、节点、行数、数据源)、哪条合并 SQL、得到几行、有哪些警告。 + # 运行详情据此展示这一步;完整结果在 query_artifact 指向的查询快照里 + MERGE_END = "merge.end" SANDBOX_START = "sandbox.start" SANDBOX_END = "sandbox.end" diff --git a/backend/app/data/engine.py b/backend/app/data/engine.py index f8109720..55c037b9 100644 --- a/backend/app/data/engine.py +++ b/backend/app/data/engine.py @@ -277,6 +277,15 @@ def open_checked_sqlite(path: str, *, readonly: bool = True, immutable: bool = F factory=_DqsOffConnection) +def open_memory_sqlite() -> sqlite3.Connection: + """一个内存 SQLite 连接,DQS 已关(合并查询把几次查询的结果落进来再合并,engine/merge_query.py)。 + + 合并 SQL 同样是模型写的:双引号里的列名写错一个字,默认连接会把它当成字符串常量,`SUM("销售颔")` 得 0, + `ON s."门店" = v."门店 "` 一行都对不上却不报错。所以和连接层共用 _DqsOffConnection,关不掉就不交出连接。 + """ + return sqlite3.connect(":memory:", factory=_DqsOffConnection) + + def _sqlite_extra() -> dict[str, Any]: """每个 SQLite engine 都带的连接参数:关 DQS。""" return {"connect_args": {"factory": _DqsOffConnection}} diff --git a/backend/app/engine/compiler.py b/backend/app/engine/compiler.py index 07c70658..05ef4472 100644 --- a/backend/app/engine/compiler.py +++ b/backend/app/engine/compiler.py @@ -11,7 +11,7 @@ from langgraph.graph import END, START, StateGraph from app.core.errors import describe_exception, raw_detail from app.core.events import EventType from app.engine.context import NodeContext, NodeError, RunContext -from app.engine.nodes import control, human, io, knowledge, llm, metrics, multi, report, tools +from app.engine.nodes import control, human, io, knowledge, llm, merge, metrics, multi, report, tools from app.engine.schema import GraphNode, GraphSpec, NodeType, back_edges, innermost_loops from app.engine.state import GraphState @@ -28,6 +28,7 @@ RUNNERS: dict[NodeType, NodeRunner] = { NodeType.SUBGRAPH: multi.run_subgraph, NodeType.TOOL: tools.run_tool, NodeType.CODE: tools.run_code, + NodeType.MERGE: merge.run_merge, NodeType.BRANCH: control.run_branch, NodeType.LOOP: control.run_loop, NodeType.HUMAN: human.run_human, diff --git a/backend/app/engine/governance.py b/backend/app/engine/governance.py index 9bdd07d3..1b07186d 100644 --- a/backend/app/engine/governance.py +++ b/backend/app/engine/governance.py @@ -237,7 +237,7 @@ REPORT_POLICY_ALSO: dict[str, tuple[str, ...]] = {"claims": (CLAIMS_JUDGE,)} JUDGE_UNLIMITED = "不设上限,费用只受句数、时长上限和每日上限约束" #: 产出本身就是证据、或者由系统确定地算出来的节点。成果字段取它们的值,不算「别的节点写的字」 -_TRACEABLE = (NodeType.REPORT, NodeType.METRICS, NodeType.INPUT, NodeType.TOOL, NodeType.RETRIEVE) +_TRACEABLE = (NodeType.REPORT, NodeType.METRICS, NodeType.INPUT, NodeType.TOOL, NodeType.RETRIEVE, NodeType.MERGE) _WRITERS = (NodeType.LLM, NodeType.AGENT, NodeType.SUPERVISOR) #: 往对话里写消息的节点(emit_message 没关时),「最后一条消息」就是它们里最后跑的那个写的 _EMITTERS = (*_WRITERS, NodeType.REPORT) diff --git a/backend/app/engine/labels.py b/backend/app/engine/labels.py index e16ead9f..073934e2 100644 --- a/backend/app/engine/labels.py +++ b/backend/app/engine/labels.py @@ -18,7 +18,7 @@ from typing import Any #: 节点类型的叫法,和前端 lib/terms.ts 的 NODE_TYPE_LABEL 一致 TYPE_LABEL: dict[str, str] = { "input": "输入", "output": "成果", "llm": "模型调用", "agent": "Agent", - "supervisor": "多 Agent 协作", "tool": "调用工具", "code": "沙箱代码", + "supervisor": "多 Agent 协作", "tool": "调用工具", "code": "沙箱代码", "merge": "合并查询", "branch": "条件分支", "loop": "循环", "subgraph": "子工作流", "memory": "长期记忆", "retrieve": "知识检索", "transform": "数据整形", "human": "人工审批", "validate": "结构校验", "metrics": "口径卡", "report": "报告撰写", @@ -79,6 +79,7 @@ _FIELD_BY_TYPE: dict[str, dict[str, str]] = { "human": {"mode": "审批方式", "title": "标题", "message": "展示给审批人的内容", "draft": "草稿内容", "stop_on_reject": "驳回即终止运行"}, "validate": {"source": "待校验内容", "fail_fast": "校验失败即中断"}, + "merge": {"inputs": "输入", "sql": "合并 SQL"}, } #: 出具契约里的键(canvas/issues.ts 的 CONTRACT_KEYS、SUB_KEYS) diff --git a/backend/app/engine/merge_query.py b/backend/app/engine/merge_query.py new file mode 100644 index 00000000..4d473886 --- /dev/null +++ b/backend/app/engine/merge_query.py @@ -0,0 +1,879 @@ +"""合并查询(merge 节点):几次查询的完整结果落进内存 SQLite,再用一条 SELECT / WITH 按键合并。 + +**为什么要有它。** 跨查询组合原来只有三种:口径卡用 cell() 组合几个单值;tool 节点的 SQL 里写 {{ vars.x }},把前一条 +查询的结果代进后一条;同一个库里写一条 JOIN / WITH。按行合并两次查询,尤其数据在两个库里时,没有路可走:用沙箱代码 +合并,出处链断在代码节点上,喂给口径卡时只能提示「核对不了出处」;让模型把 id 从一条查询抄进下一条,会抄错,结果被 +截断后还会漏。 + +**为什么不做跨库联邦查询。** 联邦查询要把两边的明细拉到一处,数据量、方言、权限一样都控制不住。方案是先在各自的库里 +聚合到共同的键和粒度(日期 + 门店),再由这里在库外按键合并:两边进来的都是已经聚合过的小结果。 + +**为什么落成 SQLite、用标准库 sqlite3。** 照搬 data/tabular.py:SQLite 是现成的 SQL 引擎,守卫有它的读法(guard +的 _SQLITE),证据下钻的识别器(direct_select)也按它分词,不必为这一步引入 DuckDB 再多一种方言。连接同样关掉 DQS +(data.engine.open_memory_sqlite):双引号里的列名写错要报错,不能当成字符串常量。 + +**守卫和三道防护。** +- SQL 先过守卫(SQLite 读法),只认 SELECT 和 WITH;表落完之后连接设成 query_only,再挂授权回调,只放行读表、 + 调函数、递归公用表表达式。守卫是第一道,后两道不赌守卫没有漏洞; +- 结果的行数、字节上限和查询层相同(QueryLimits),超时按查询层的缺省时限由进度回调中断; +- 任一输入被截断 → 拒绝合并:拿只有前若干行的结果按键合并,缺的行不报错,只会悄悄少算; +- 合并键两边类型不一致(文本 '001' 对数值 1)→ 警告。SQLite 比较时按亲和性隐式转换,'001'、'01' 都会等于 1; +- 结果行数多于行数最多的输入 → 警告合并键可能不唯一(多对多匹配把行放大了)。 + +**逐格来历:宁可不下钻,也不下钻错。** 报告里引用合并结果的一格,证据面板要能指回它来自哪个输入的哪一格。只有一种 +写法认得出来:单层 SELECT,FROM 后面是输入表(可加别名、可 JOIN),选取项是某张输入表的列(可改名);没有分组、 +聚合、DISTINCT、子查询、公用表表达式、集合运算、窗口和 NATURAL JOIN。认出来以后,在原 SQL 的选取列表末尾补上每张 +输入表的 rowid 再执行一次:补列的结果去掉补的几列,必须和原结果逐行逐值相同,才把 rowid 记成来历;对不上整份不记。 +另外两种情况也不给:结果里一模一样的两行(说不清哪一行来自哪一行输入),以及来历那一格和结果的值对不上的列。 +表达式、聚合出来的列没有逐格来历,只有表级来历:来自哪几个输入、哪条合并 SQL。 + +纯函数:不连数据库、不读写工件、不发事件。取上游快照、存快照、发事件在执行器(engine/nodes/merge.py)。 +""" +from __future__ import annotations + +import json +import re +import sqlite3 +import time +from collections import Counter +from dataclasses import dataclass, field +from typing import Any + +from app.data import guard +from app.data.engine import _jsonable, column_types, open_memory_sqlite +from app.data.guard import QueryLimits, SqlRejected +from app.data.names import name_key +from app.engine.direct_select import _kw, _lex, _Lex +from app.engine.expressions import same_value + +#: 合并查询结果的查询快照里 source 记这个。数据源名只能是小写英文、数字和下划线(api/datasources.py 的 +#: _NAME_PATTERN),永远撞不上它;证据面板、裁判摘录据此认出合并结果,不去按数据源名找遮罩 +MERGE_SOURCE = "合并查询" +#: 一个合并节点最多几个输入。再多通常说明该在源库里先合并 +MAX_INPUTS = 8 +#: 别名(也就是内存库里的表名)最长几个字符 +ALIAS_MAX = 32 + +_ALIAS = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +#: SQLite 的全部关键字(sqlite.org/lang_keywords.html),加上 true / false。用作表名要加引号才读得对, +#: 模型写合并 SQL 时多半不加——不如在起名时就挡住 +_SQLITE_KEYWORDS = frozenset(""" +abort action add after all alter always analyze and as asc attach autoincrement before begin between by cascade case +cast check collate column commit conflict constraint create cross current current_date current_time current_timestamp +database default deferrable deferred delete desc detach distinct do drop each else end escape except exclude exclusive +exists explain fail filter first following for foreign from full generated glob group groups having if ignore immediate +in index indexed initially inner insert instead intersect into is isnull join key last left like limit match +materialized natural no not nothing notnull null nulls of offset on or order others outer over partition plan pragma +preceding primary query raise range recursive references regexp reindex release rename replace restrict returning +right rollback row rows savepoint select set table temp temporary then ties to transaction trigger unbounded union +unique update using vacuum values view virtual when where window with without true false +""".split()) + +#: 快照里记的列类型 → 内存表的列亲和性。number 用 NUMERIC:数据源把 DECIMAL 落成的文本("3200.50")进来就是数, +#: 能直接参与运算;文本、日期一律 TEXT,'007' 的前导零不丢。拿不准的列不写类型,值原样存 +_AFFINITY = {"number": "NUMERIC", "boolean": "INTEGER", "text": "TEXT", "date": "TEXT", "datetime": "TEXT", + "time": "TEXT"} +_KIND_LABEL = {"number": "数值", "boolean": "布尔值", "text": "文本", "date": "日期", "datetime": "日期时间", + "time": "时间"} +#: SQLite 表的 rowid 有三个名字;输入里恰好有同名的列时换下一个,三个都占了就不追逐格来历 +_ROWID_NAMES = ("rowid", "_rowid_", "oid") +#: 补进选取列表的来历列。带引号写,和输入的列名撞不上(输入的列名里不会有这个前缀,撞了也只是对不上、不记来历) +_LINEAGE_COLUMN = "__agentlab_merge_row_{}" + +SELECT_ONLY = "合并 SQL 只能是 SELECT 或 WITH 查询" +_TEMPLATE = re.compile(r"\{\{.*?\}\}", re.S) + +#: 授权回调放行的动作:读表、调函数、递归公用表表达式。别的一律拒(ATTACH、PRAGMA、写) +_ALLOWED_ACTIONS = frozenset({sqlite3.SQLITE_SELECT, sqlite3.SQLITE_READ, sqlite3.SQLITE_FUNCTION, + sqlite3.SQLITE_RECURSIVE}) + + +class MergeError(ValueError): + """合并做不成。消息直接给人看:哪个输入、为什么、怎么改。""" + + +@dataclass(frozen=True) +class MergeInput: + """一个输入:别名(内存库里的表名)、上游节点、它的查询快照。""" + + alias: str + node_id: str + #: 上游查询快照的工件 id(tool 节点的 query_snapshot,或另一个合并节点的结果快照) + artifact: str + snapshot: dict[str, Any] + #: 上游节点在画布上的名字,只用在报错里 + label: str = "" + + +@dataclass(frozen=True) +class Trace: + """合并结果的一格来自哪个输入的哪一格(只走一跳;输入本身是合并结果时由调用方再追)。""" + + alias: str + node_id: str | None + artifact: str | None + row: int + column: str + + +@dataclass +class MergeOutcome: + columns: list[str] + rows: list[list[Any]] + truncated: bool + #: 守卫规范过的那一条 SQL,执行的就是它 + sql: str + column_types: dict[str, str] + warnings: list[dict[str, Any]] + #: 逐格来历(见模块说明);认不出写法、核对不上时为 None + lineage: dict[str, Any] | None + inputs: list[dict[str, Any]] + #: 底层数据源(输入本身是合并结果的,展开成它的数据源),去重保序 + sources: list[str] + mask_columns: list[str] = field(default_factory=list) + + def snapshot(self) -> dict[str, Any]: + """存进工件库的查询快照:和数据源查询同形(columns、rows、row_count、truncated、sql、column_types、source), + 另记 inputs、sources、warnings、lineage。不含耗时之类每次不同的东西:同样的输入、同样的 SQL 得到同一个工件, + 节点重放、继续运行时快照 id 不变。""" + out: dict[str, Any] = { + "columns": self.columns, "rows": self.rows, "row_count": len(self.rows), "truncated": self.truncated, + "sql": self.sql, "source": MERGE_SOURCE, "inputs": self.inputs, "sources": self.sources, + "warnings": self.warnings, + } + if self.column_types: + out["column_types"] = self.column_types + if self.lineage is not None: + out["lineage"] = self.lineage + if self.mask_columns: + out["mask_columns"] = self.mask_columns + return out + + +# -------------------------------------------------------------------------- +# 静态检查:别名、SQL(画图时校验用,执行时同一套规则) +# -------------------------------------------------------------------------- + + +def alias_problem(alias: Any) -> str | None: + """别名能不能当内存库里的表名。能用返回 None。""" + if not isinstance(alias, str) or not alias: + return "别名不能为空" + if len(alias) > ALIAS_MAX: + return f"别名「{alias[:ALIAS_MAX]}…」太长,最多 {ALIAS_MAX} 个字符" + if not _ALIAS.match(alias): + return f"别名「{alias}」不是合法的表名:只能用英文字母、数字和下划线,且不能以数字开头" + if alias.lower() in _SQLITE_KEYWORDS: + return f"别名「{alias}」是 SQL 关键字,用作表名必须加引号,容易写错。请换一个,例如 sales、visits" + if alias.lower().startswith("sqlite_"): + return f"别名「{alias}」以 sqlite_ 开头,这是 SQLite 保留给内部表的名字,请换一个" + return None + + +def sql_problem(sql: Any) -> str | None: + """合并 SQL 有什么问题,没问题返回 None。{{ }} 运行时才有值,先按一个空值代进去检查写法。""" + if not isinstance(sql, str) or not sql.strip(): + return "还没有填写合并 SQL" + try: + _checked(_TEMPLATE.sub("NULL", sql)) + except MergeError as e: + return str(e) + return None + + +def _checked(sql: str) -> str: + """守卫(SQLite 读法)规范过的那一条语句。只认 SELECT / WITH;守卫的原话写给数据源的,换成合并查询自己的说法。""" + if not sql or not sql.strip(): + raise MergeError("还没有填写合并 SQL") + statements = guard.split_statements(sql, dialect="sqlite") + if len(statements) > 1: + raise MergeError(f"合并 SQL 只能写一条语句,这里有 {len(statements)} 条。请把它们合成一条 SELECT(可以用 WITH)") + try: + stmt = guard.check(sql, readonly=True, dialect="sqlite") + except SqlRejected as e: + # 只读拒绝的原话是「请在数据页另建一个可写的数据源」,合并查询用不上;结构变更(ATTACH、DROP)同理 + if guard.is_write(sql, dialect="sqlite"): + raise MergeError(f"{SELECT_ONLY},不能写入、建表或连接别的库") from None + raise MergeError(f"合并 SQL 无法执行:{e}") from None + verb = guard.first_verb(stmt, dialect="sqlite") + if verb not in ("select", "with"): + raise MergeError(f"{SELECT_ONLY},当前是 {verb.upper() or '无法识别的语句'}") + return stmt + + +# -------------------------------------------------------------------------- +# 执行 +# -------------------------------------------------------------------------- + + +def execute(inputs: list[MergeInput], sql: str, *, limits: QueryLimits | None = None) -> MergeOutcome: + """把输入落进内存库,执行合并 SQL。做不成抛 MergeError(消息给人看)。""" + limits = limits or QueryLimits() + if not inputs: + raise MergeError("还没有配置输入:至少选择一个上游的查询节点") + if len(inputs) > MAX_INPUTS: + raise MergeError(f"输入最多 {MAX_INPUTS} 个,这里有 {len(inputs)} 个。请先在源库里合并一部分") + _check_aliases(inputs) + tables = {inp.alias: _input_columns(inp) for inp in inputs} + stmt = _checked(sql) + + conn = open_memory_sqlite() + try: + rowids = {inp.alias: _load(conn, inp, tables[inp.alias]) for inp in inputs} + # 表落完就只读:授权回调之外的又一道(守卫、授权回调任何一处有漏洞,写也写不进去) + conn.execute("PRAGMA query_only = 1") + conn.set_authorizer(_authorize) + deadline = time.monotonic() + float(limits.timeout_seconds) + conn.set_progress_handler(lambda: 1 if time.monotonic() > deadline else 0, 1000) + columns, rows, truncated = _read(conn, stmt, limits) + plan = _lineage_plan(stmt, inputs, tables, rowids) + lineage = _lineage(conn, plan, columns, rows, inputs, tables) if plan is not None else None + except sqlite3.Error as e: + raise MergeError(_sqlite_reason(e, limits)) from None + finally: + conn.close() + + warnings = _warnings(stmt, inputs, tables, rows, truncated, limits) + return MergeOutcome( + columns=columns, rows=rows, truncated=truncated, sql=stmt, column_types=column_types(columns, rows), + warnings=warnings, lineage=lineage, inputs=[_summary(inp, tables[inp.alias]) for inp in inputs], + sources=_sources(inputs), mask_columns=_masks(inputs, columns, plan), + ) + + +def _check_aliases(inputs: list[MergeInput]) -> None: + seen: dict[str, str] = {} + for inp in inputs: + if problem := alias_problem(inp.alias): + raise MergeError(problem) + key = inp.alias.lower() + if key in seen: + raise MergeError(f"别名「{seen[key]}」和「{inp.alias}」只差大小写:SQLite 的表名不分大小写,会被当成同一张表。" + "请换一个别名") + seen[key] = inp.alias + + +def _who(inp: MergeInput) -> str: + return f"输入「{inp.alias}」" + (f"({inp.label})" if inp.label and inp.label != inp.alias else "") + + +def _input_columns(inp: MergeInput) -> list[str]: + snapshot = inp.snapshot + if not isinstance(snapshot, dict) or not isinstance(snapshot.get("columns"), list) \ + or not isinstance(snapshot.get("rows"), list): + raise MergeError(f"{_who(inp)}不是查询结果:快照里没有列和行") + if snapshot.get("truncated"): + raise MergeError( + f"{_who(inp)}的查询结果被截断,只拿到了前 {len(snapshot['rows'])} 行。拿不完整的结果按键合并,缺的行不会报错," + "只会让合并结果悄悄少算。请在源库里先聚合到合并需要的粒度(例如按日期和门店 GROUP BY)," + "或者加条件缩小范围,让结果不超过查询上限") + columns = [str(c) for c in snapshot["columns"]] + if not columns: + raise MergeError(f"{_who(inp)}的查询结果没有列") + seen: dict[str, str] = {} + for name in columns: + key = name_key(name) + if key in seen: + raise MergeError(f"{_who(inp)}的结果里有重名的列「{name}」:SQLite 的列名不分大小写,同名的列放不进一张表。" + "请在源查询中用 AS 给它们起不同的名字") + seen[key] = name + return columns + + +def _quote(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def _kind(snapshot: dict[str, Any], column: str) -> str | None: + """这一列的类型:查询时记下的(column_types)优先;老快照没记的按值猜,猜不准不说。""" + types = snapshot.get("column_types") + if isinstance(types, dict) and isinstance(types.get(column), str): + return types[column] + columns = [str(c) for c in snapshot.get("columns") or []] + if column not in columns: + return None + at = columns.index(column) + kinds = {_value_kind(row[at]) for row in snapshot.get("rows") or [] + if isinstance(row, list) and at < len(row) and row[at] is not None} + return next(iter(kinds)) if len(kinds) == 1 and None not in kinds else None + + +def _value_kind(value: Any) -> str | None: + if isinstance(value, bool): + return "boolean" + if isinstance(value, (int, float)): + return "number" + return "text" if isinstance(value, str) else None + + +def _cell_in(value: Any) -> Any: + """快照里的值 → 写进内存库的值。快照是 JSON:布尔写成 0 / 1,对象和数组写成 JSON 文本。""" + if value is None or isinstance(value, (str, int, float)) and not isinstance(value, bool): + return value + if isinstance(value, bool): + return int(value) + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False, sort_keys=True) + return str(value) + + +def _load(conn: sqlite3.Connection, inp: MergeInput, columns: list[str]) -> str | None: + """建表、按快照的顺序写入行,返回这张表的 rowid 用哪个名字(第 i 行的 rowid 是 i + 1)。""" + snapshot = inp.snapshot + decl = ", ".join(f"{_quote(c)} {_AFFINITY.get(_kind(snapshot, c) or '', '')}".rstrip() for c in columns) + conn.execute(f"CREATE TABLE {_quote(inp.alias)} ({decl})") + taken = {c.lower() for c in columns} + rowid = next((n for n in _ROWID_NAMES if n not in taken), None) + width = len(columns) + rows: list[list[Any]] = [] + for i, row in enumerate(snapshot["rows"]): + if isinstance(row, dict): + row = [row.get(c) for c in columns] + if not isinstance(row, list) or len(row) != width: + raise MergeError(f"{_who(inp)}的快照第 {i} 行和列数对不上,快照可能已损坏") + rows.append([i + 1, *map(_cell_in, row)] if rowid else [_cell_in(v) for v in row]) + names = ", ".join([*([rowid] if rowid else []), *map(_quote, columns)]) + marks = ", ".join("?" * (width + (1 if rowid else 0))) + conn.executemany(f"INSERT INTO {_quote(inp.alias)} ({names}) VALUES ({marks})", rows) + return rowid + + +def _authorize(action: int, *_args: Any) -> int: + return sqlite3.SQLITE_OK if action in _ALLOWED_ACTIONS else sqlite3.SQLITE_DENY + + +def _read(conn: sqlite3.Connection, stmt: str, limits: QueryLimits) -> tuple[list[str], list[list[Any]], bool]: + """逐行读、边读边判上限:和查询层(data.engine.run_query)同一个口径,到了行数或字节上限就停,记 truncated。""" + cursor = conn.execute(stmt) + columns = [str(d[0]) for d in cursor.description or []] + if not columns: + raise MergeError("合并 SQL 没有返回任何列") + rows: list[list[Any]] = [] + size = 0 + truncated = False + for row in cursor: + values = [_jsonable(v) for v in row] + rows.append(values) + size += len(json.dumps(values, ensure_ascii=False, default=str)) + if len(rows) >= limits.max_rows or size >= limits.max_bytes: + truncated = True + break + return columns, rows, truncated + + +def _sqlite_reason(e: sqlite3.Error, limits: QueryLimits) -> str: + text = str(e) + if "interrupted" in text: + seconds = float(limits.timeout_seconds) + shown = f"{seconds:g}" + return f"合并超过 {shown} 秒被中断。请在源查询里先聚合,或在合并 SQL 中加条件缩小范围" + if "not authorized" in text or "readonly" in text: + return f"{SELECT_ONLY},只能读取输入表" + # SQLite 的原话(no such column: x、ambiguous column name: 日期)正是改 SQL 要的线索,原样带上 + return f"合并 SQL 执行失败:{text}" + + +def _summary(inp: MergeInput, columns: list[str]) -> dict[str, Any]: + source = inp.snapshot.get("source") + return {"alias": inp.alias, "node_id": inp.node_id, "artifact": inp.artifact, "rows": len(inp.snapshot["rows"]), + "columns": columns, "source": str(source) if source else None} + + +def _sources(inputs: list[MergeInput]) -> list[str]: + out: list[str] = [] + for inp in inputs: + nested = inp.snapshot.get("sources") + names = nested if inp.snapshot.get("source") == MERGE_SOURCE and isinstance(nested, list) \ + else [inp.snapshot.get("source")] + out.extend(str(n) for n in names if n and str(n) not in out) + return out + + +# -------------------------------------------------------------------------- +# 逐格来历 +# -------------------------------------------------------------------------- + +#: 整条语句里出现就不追来历的词:子查询、集合运算、公用表表达式、分组、窗口、去重、NATURAL JOIN +_NO_LINEAGE = frozenset({"select", "union", "intersect", "except", "with", "recursive", "group", "having", "window", + "over", "distinct", "values", "natural", "filter"}) +#: 聚合函数(名字后面紧跟「(」)。没有 GROUP BY 的聚合把整张表压成一行,选取项里的裸列取的是哪一行说不准 +_AGGREGATES = frozenset({"count", "sum", "avg", "min", "max", "total", "group_concat", "string_agg", "median", + "json_group_array", "json_group_object", "jsonb_group_array", "jsonb_group_object"}) +#: 不加引号的字面量关键字:选取项写 NULL、CURRENT_DATE 不是列 +_LITERALS = frozenset({"null", "true", "false", "current_date", "current_time", "current_timestamp"}) +#: 来源表后面紧跟这些词时,它们不是省略了 AS 的表别名 +_AFTER_SOURCE = frozenset({"on", "using", "where", "order", "limit", "join", "left", "right", "full", "inner", "cross", + "outer", "indexed", "not", "natural", "group", "having", "window", "union", "intersect", + "except", "as"}) +_JOIN_WORDS = frozenset({"join", "left", "right", "full", "inner", "cross", "natural"}) +_TAIL = frozenset({"where", "order", "limit"}) + + +@dataclass(frozen=True) +class _Source: + #: 输入别名(内存库里的表名) + alias: str + #: SQL 里怎么称呼它:写了表别名就是表别名,否则是表名本身 + ref: str + + +@dataclass(frozen=True) +class _Plan: + sources: list[_Source] + #: 第 i 个结果列:(第几个来源, 输入里的列名);表达式、说不准的列是 None + columns: list[tuple[int, str] | None] + #: 在原 SQL 的哪个位置补来历列(顶层 FROM 前面) + insert_at: int + stmt: str + rowids: list[str] + + +def _punct(lex: _Lex | None, text: str) -> bool: + return lex is not None and lex.kind == "punct" and lex.text == text + + +def _named(lex: _Lex | None) -> bool: + return lex is not None and lex.kind in ("word", "name") + + +def _find(names: list[str], wanted: str) -> str | None: + key = name_key(wanted) + return next((n for n in names if name_key(n) == key), None) + + +def _lineage_plan(stmt: str, inputs: list[MergeInput], tables: dict[str, list[str]], + rowids: dict[str, str | None]) -> _Plan | None: + """认出「单层 SELECT、选取项是输入表的列」这种写法;认不出返回 None(整份不记来历)。""" + toks = _lex(stmt) + if not toks or _kw(toks[0]) != "select": + return None + if any(_kw(t) in _NO_LINEAGE for t in toks[1:]): + return None + if any(_kw(t) in _AGGREGATES and _punct(nxt, "(") for t, nxt in zip(toks, toks[1:])): + return None + depth, from_at = 0, None + for i, t in enumerate(toks): + if _punct(t, "("): + depth += 1 + elif _punct(t, ")"): + depth -= 1 + elif depth == 0 and _kw(t) == "from": + from_at = i + break + if from_at is None: + return None + start = 2 if _kw(toks[1]) == "all" else 1 + parsed = _from_clause(toks, from_at + 1, tables) + if parsed is None: + return None + sources, using = parsed + columns: list[tuple[int, str] | None] = [] + for item in _split_items(toks[start:from_at]): + mapped = _item(item, sources, tables, using) + if mapped is None: + return None + columns.extend(mapped) + if any(rowids.get(s.alias) is None for s in sources): + return None + frm = toks[from_at] + return _Plan(sources=sources, columns=columns, insert_at=frm.end - len(frm.text), stmt=stmt, + rowids=[str(rowids[s.alias]) for s in sources]) + + +def _split_items(toks: list[_Lex]) -> list[list[_Lex]]: + items: list[list[_Lex]] = [[]] + depth = 0 + for t in toks: + if _punct(t, "("): + depth += 1 + elif _punct(t, ")"): + depth -= 1 + if depth == 0 and _punct(t, ","): + items.append([]) + else: + items[-1].append(t) + return items + + +def _from_clause(toks: list[_Lex], i: int, tables: dict[str, list[str]]) -> tuple[list[_Source], set[str]] | None: + """FROM 之后:来源表 [AS 别名],用逗号或 JOIN 连接,JOIN 后面可跟 ON 条件或 USING (列…);之后只许 + WHERE / ORDER BY / LIMIT。来源只能是输入表(不认子查询、表函数、带库名的表)。""" + sources: list[_Source] = [] + using: set[str] = set() + n = len(toks) + while True: + t = toks[i] if i < n else None + if not _named(t) or (t.kind == "word" and name_key(t.text) in _SQLITE_KEYWORDS): + return None + table = _find(list(tables), t.text) + if table is None or _punct(toks[i + 1] if i + 1 < n else None, ".") \ + or _punct(toks[i + 1] if i + 1 < n else None, "("): + return None + ref, i = t.text, i + 1 + nxt = toks[i] if i < n else None + if _kw(nxt) == "as": + if not _named(toks[i + 1] if i + 1 < n else None): + return None + ref, i = toks[i + 1].text, i + 2 + elif _named(nxt) and (nxt.kind == "name" or name_key(nxt.text) not in _AFTER_SOURCE): + ref, i = nxt.text, i + 1 + sources.append(_Source(alias=table, ref=ref)) + word = _kw(toks[i]) if i < n else None + if word == "on": + depth, i = 0, i + 1 + while i < n: + t = toks[i] + if _punct(t, "("): + depth += 1 + elif _punct(t, ")"): + depth -= 1 + elif depth == 0 and (_punct(t, ",") or _kw(t) in _JOIN_WORDS or _kw(t) in _TAIL): + break + i += 1 + elif word == "using": + if not _punct(toks[i + 1] if i + 1 < n else None, "("): + return None + i += 2 + while i < n and not _punct(toks[i], ")"): + if _named(toks[i]): + using.add(name_key(toks[i].text)) + elif not _punct(toks[i], ","): + return None + i += 1 + i += 1 + if i >= n: + return sources, using + t = toks[i] + if _punct(t, ","): + i += 1 + continue + word = _kw(t) + if word in _TAIL: + return sources, using + if word not in ("join", "left", "right", "full", "inner", "cross"): + return None + while i < n and _kw(toks[i]) in ("left", "right", "full", "inner", "cross", "outer"): + i += 1 + if i >= n or _kw(toks[i]) != "join": + return None + i += 1 + + +def _source_of(ref: str, sources: list[_Source]) -> int | None: + key = name_key(ref) + hits = [k for k, s in enumerate(sources) if name_key(s.ref) == key] + return hits[0] if len(hits) == 1 else None + + +def _item(item: list[_Lex], sources: list[_Source], tables: dict[str, list[str]], + using: set[str]) -> list[tuple[int, str] | None] | None: + """一个选取项展开成几个结果列的来历。返回 None 表示整份不追(* 遇上 USING:SQLite 只出一份合并掉的列)。""" + if len(item) == 1 and _punct(item[0], "*"): + if using: + return None + return [(k, c) for k, s in enumerate(sources) for c in tables[s.alias]] + if len(item) == 3 and _named(item[0]) and _punct(item[1], ".") and _punct(item[2], "*"): + k = _source_of(item[0].text, sources) + return None if k is None else [(k, c) for c in tables[sources[k].alias]] + body = item + if len(item) >= 3 and _kw(item[-2]) == "as" and _named(item[-1]): + body = item[:-2] + elif len(item) >= 2 and _named(item[-1]) and _named(item[-2]) \ + and not (item[-1].kind == "word" and name_key(item[-1].text) in _SQLITE_KEYWORDS): + body = item[:-1] + if len(body) == 1 and _named(body[0]): + word = body[0] + if word.kind == "word" and name_key(word.text) in _LITERALS: + return [None] + owners = [(k, col) for k, s in enumerate(sources) if (col := _find(tables[s.alias], word.text)) is not None] + if len(owners) != 1 or name_key(word.text) in using: + return [None] + return [owners[0]] + if len(body) == 3 and _named(body[0]) and _punct(body[1], ".") and _named(body[2]): + k = _source_of(body[0].text, sources) + col = _find(tables[sources[k].alias], body[2].text) if k is not None else None + return [(k, col) if k is not None and col is not None else None] + return [None] + + +def _hashable(row: list[Any]) -> str: + return json.dumps(row, ensure_ascii=False, sort_keys=True, default=str) + + +def _lineage(conn: sqlite3.Connection, plan: _Plan, columns: list[str], rows: list[list[Any]], + inputs: list[MergeInput], tables: dict[str, list[str]]) -> dict[str, Any] | None: + """补上 rowid 再执行一次,逐行逐值核对,得出每一格来自哪个输入的哪一行。核对不上返回 None。""" + width = len(columns) + if len(plan.columns) != width or not any(plan.columns): + return None + extra = ", ".join(f"{_quote(s.ref)}.{rowid} AS {_quote(_LINEAGE_COLUMN.format(k))}" + for k, (s, rowid) in enumerate(zip(plan.sources, plan.rowids))) + head = plan.stmt[:plan.insert_at].rstrip() + try: + augmented = _checked(f"{head}, {extra} {plan.stmt[plan.insert_at:]}") + cursor = conn.execute(augmented) + except (MergeError, sqlite3.Error): + return None + got: list[tuple[Any, ...]] = [] + for row in cursor: + if len(got) >= len(rows): + break + got.append(row) + if len(got) != len(rows) or any(len(g) != width + len(plan.sources) for g in got): + return None + if any([_jsonable(v) for v in g[:width]] != r for g, r in zip(got, rows)): + return None + origin: list[list[int | None]] = [[v - 1 if isinstance(v, int) and v >= 1 else None for v in g[width:]] + for g in got] + # 一模一样的两行结果:说不清哪一行来自哪一行输入 + counts = Counter(_hashable(r) for r in rows) + for i, r in enumerate(rows): + if counts[_hashable(r)] > 1: + origin[i] = [None] * len(plan.sources) + snaps = {inp.alias: inp.snapshot for inp in inputs} + mapped: list[dict[str, Any] | None] = [] + for j, target in enumerate(plan.columns): + if target is None: + mapped.append(None) + continue + k, column = target + snap = snaps[plan.sources[k].alias] + at = tables[plan.sources[k].alias].index(column) + ok = True + for i, row in enumerate(rows): + src = origin[i][k] + if src is None: + continue + if not (0 <= src < len(snap["rows"])) or not _same(row[j], _row(snap["rows"][src], tables[plan.sources[k].alias])[at]): + ok = False + break + mapped.append({"source": k, "column": column} if ok else None) + if not any(mapped): + return None + return {"sources": [{"input": s.alias, "ref": s.ref} for s in plan.sources], "columns": mapped, "rows": origin} + + +def _row(row: Any, columns: list[str]) -> list[Any]: + return [row.get(c) for c in columns] if isinstance(row, dict) else row + + +def _same(merged: Any, original: Any) -> bool: + if merged is None or original is None: + return merged is None and original is None + return same_value(merged, _cell_in(original)) + + +def trace_cell(snapshot: Any, row: Any, column: Any) -> Trace | None: + """合并结果的这一格来自哪个输入的哪一格;没有逐格来历返回 None。列按第一次出现的位置认(同 locate_cell)。""" + if not isinstance(snapshot, dict) or snapshot.get("source") != MERGE_SOURCE: + return None + lineage, columns = snapshot.get("lineage"), snapshot.get("columns") + if not isinstance(lineage, dict) or not isinstance(columns, list) or column not in columns: + return None + if isinstance(row, bool) or not isinstance(row, int): + return None + mapped, origin, sources = lineage.get("columns"), lineage.get("rows"), lineage.get("sources") + at = columns.index(column) + if not (isinstance(mapped, list) and isinstance(origin, list) and isinstance(sources, list) + and at < len(mapped) and 0 <= row < len(origin)): + return None + target = mapped[at] + if not isinstance(target, dict) or not isinstance(target.get("source"), int): + return None + k = target["source"] + line = origin[row] + if not isinstance(line, list) or not 0 <= k < len(line) or not 0 <= k < len(sources): + return None + src = line[k] + alias = (sources[k] or {}).get("input") if isinstance(sources[k], dict) else None + if isinstance(src, bool) or not isinstance(src, int) or not isinstance(alias, str): + return None + entry = next((e for e in snapshot.get("inputs") or [] if isinstance(e, dict) and e.get("alias") == alias), None) + if entry is None: + return None + return Trace(alias=alias, node_id=entry.get("node_id"), artifact=entry.get("artifact"), row=src, + column=str(target.get("column"))) + + +def _masks(inputs: list[MergeInput], columns: list[str], plan: _Plan | None) -> list[str]: + """合并结果要遮罩的列:各输入当时记下的遮罩列,加上来自遮罩列、换了名字的结果列(宁可多遮)。""" + hidden: dict[str, set[str]] = {} + names: list[str] = [] + for inp in inputs: + recorded = inp.snapshot.get("mask_columns") + cols = {str(c).lower() for c in recorded} if isinstance(recorded, list) else set() + hidden[inp.alias] = cols + names.extend(str(c) for c in recorded or [] if isinstance(c, str)) + if plan is not None and len(plan.columns) == len(columns): + for name, target in zip(columns, plan.columns): + if target is not None and target[1].lower() in hidden.get(plan.sources[target[0]].alias, set()): + names.append(name) + return sorted(dict.fromkeys(names)) + + +# -------------------------------------------------------------------------- +# 警告:合并键类型、行数放大、结果截断 +# -------------------------------------------------------------------------- + + +def _key_pairs(stmt: str, tables: dict[str, list[str]]) -> list[tuple[str, str, str, str]]: + """合并条件里的键:ON / WHERE 里「列 = 列」两边落在不同的输入表上,以及 USING (列)。只用来提示,认不全不要紧。""" + toks = _lex(stmt, strict=False) or [] + n = len(toks) + refs: dict[str, str] = {} + pairs: list[tuple[str, str, str, str]] = [] + placed: list[str] = [] # 当前 FROM 列表里已经出现过的输入表(USING 找左边那张) + i = 0 + while i < n: + word = _kw(toks[i]) + if word in ("from", "join") or (_punct(toks[i], ",") and placed and i > 0 and _named(toks[i - 1])): + if word == "from": + placed = [] + j = i + 1 + t = toks[j] if j < n else None + table = _find(list(tables), t.text) if _named(t) else None + if table is not None and not _punct(toks[j + 1] if j + 1 < n else None, "."): + ref, j = t.text, j + 1 + nxt = toks[j] if j < n else None + if _kw(nxt) == "as" and _named(toks[j + 1] if j + 1 < n else None): + ref, j = toks[j + 1].text, j + 2 + elif _named(nxt) and (nxt.kind == "name" or name_key(nxt.text) not in _AFTER_SOURCE): + ref, j = nxt.text, j + 1 + refs[name_key(ref)] = table + refs.setdefault(name_key(table), table) + if _kw(toks[j] if j < n else None) == "using" and _punct(toks[j + 1] if j + 1 < n else None, "("): + k = j + 2 + while k < n and not _punct(toks[k], ")"): + if _named(toks[k]): + col = _find(tables[table], toks[k].text) + left = next((p for p in reversed(placed) if _find(tables[p], toks[k].text)), None) + if col is not None and left is not None: + pairs.append((left, str(_find(tables[left], toks[k].text)), table, col)) + k += 1 + placed.append(table) + i = j + continue + i += 1 + + def resolve(at: int, step: int) -> tuple[tuple[str, str] | None, int]: + """从 at 开始(step=1 往右、-1 往左)读一个列引用:(表, 列)、它占了几个记号。""" + if step > 0: + seq = toks[at:at + 3] + else: + seq = toks[max(0, at - 2):at + 1] + if len(seq) == 3 and _named(seq[0]) and _punct(seq[1], ".") and _named(seq[2]): + table = refs.get(name_key(seq[0].text)) + col = _find(tables[table], seq[2].text) if table else None + return ((table, col) if table and col else None), 3 + one = toks[at] if 0 <= at < n else None + if _named(one): + owners = [(t, c) for t in tables if (c := _find(tables[t], one.text)) is not None] + return (owners[0] if len(owners) == 1 else None), 1 + return None, 1 + + for i, t in enumerate(toks): + if not _punct(t, "="): + continue + prev = toks[i - 1] if i > 0 else None + # 「==」的第二个等号、<=、>=、!= 都不是比较的开头(分词把每个标点切成一个记号) + if prev is None or (prev.kind == "punct" and prev.text in "=<>!"): + continue + right_at = i + 2 if _punct(toks[i + 1] if i + 1 < n else None, "=") else i + 1 + left, width = resolve(i - 1, -1) + before = toks[i - 1 - width] if i - 1 - width >= 0 else None + if before is not None and _punct(before, "."): + left = None # a.b.c 这种带库名的写法:不是输入表的列 + right, rwidth = resolve(right_at, 1) + after = toks[right_at + rwidth] if right_at + rwidth < n else None + if after is not None and (_punct(after, ".") or _punct(after, "(")): + right = None + if left and right and left[0] != right[0]: + pairs.append((left[0], left[1], right[0], right[1])) + seen: set[frozenset[tuple[str, str]]] = set() + out: list[tuple[str, str, str, str]] = [] + for a, ac, b, bc in pairs: + key = frozenset({(a, ac), (b, bc)}) + if key not in seen: + seen.add(key) + out.append((a, ac, b, bc)) + return out + + +def _example(snapshot: dict[str, Any], column: str) -> str | None: + columns = [str(c) for c in snapshot.get("columns") or []] + if column not in columns: + return None + at = columns.index(column) + for row in snapshot.get("rows") or []: + value = row[at] if isinstance(row, list) and at < len(row) else None + if value is not None: + return f"'{value}'" if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + return None + + +def _clash(a: str | None, b: str | None) -> bool: + """两边的类型放在一起比会不会出错:数值对文本类,日期对日期时间('2026-05-01' 永远不等于 '2026-05-01T00:00:00')。""" + if not a or not b or a == b: + return False + numeric = {"number", "boolean"} + if (a in numeric) != (b in numeric): + return True + return {a, b} == {"date", "datetime"} + + +def _warnings(stmt: str, inputs: list[MergeInput], tables: dict[str, list[str]], rows: list[list[Any]], + truncated: bool, limits: QueryLimits) -> list[dict[str, Any]]: + snaps = {inp.alias: inp.snapshot for inp in inputs} + pairs = _key_pairs(stmt, tables) + out: list[dict[str, Any]] = [] + for a, ac, b, bc in pairs: + ka, kb = _kind(snaps[a], ac), _kind(snaps[b], bc) + if not _clash(ka, kb): + continue + ea, eb = _example(snaps[a], ac), _example(snaps[b], bc) + left = f"{a}.{ac} 是{_KIND_LABEL.get(str(ka), ka)}" + (f"(例如 {ea})" if ea else "") + right = f"{b}.{bc} 是{_KIND_LABEL.get(str(kb), kb)}" + (f"(例如 {eb})" if eb else "") + if {ka, kb} == {"date", "datetime"}: + tail = ("日期和日期时间按文本比较永远不相等,这组键可能一行都匹配不上。请在源查询中统一格式," + "或在合并 SQL 中用 date() 转换后再比较") + else: + tail = ("SQLite 比较时会做隐式转换:文本形式的编号(如 '001'、'01')都会等于数值 1,也可能完全匹配不上。" + "请在源查询中统一类型,或在合并 SQL 中用 CAST 明确转换") + out.append({"code": "key_type_mismatch", "keys": [f"{a}.{ac}", f"{b}.{bc}"], + "message": f"合并键类型不一致:{left},{right}。{tail}"}) + largest = max(inputs, key=lambda inp: len(inp.snapshot["rows"])) + biggest = len(largest.snapshot["rows"]) + if len(rows) > biggest: + keys: dict[str, list[str]] = {} + for a, ac, b, bc in pairs: + for alias, col in ((a, ac), (b, bc)): + if col not in keys.setdefault(alias, []): + keys[alias].append(col) + dups = [_duplicates(alias, snaps[alias], cols) for alias, cols in keys.items()] + detail = ";".join(d for d in dups if d) + out.append({"code": "rows_grew", "message": ( + f"合并结果有 {len(rows)} 行,多于行数最多的输入「{largest.alias}」({biggest} 行):合并键可能不唯一," + "同一行被重复匹配。" + (f"{detail}。" if detail else "") + + "请检查合并条件是否覆盖了全部键(例如同时按日期和门店),或先在源库里聚合到相同粒度")}) + if truncated: + out.append({"code": "result_truncated", "message": ( + f"合并结果超过上限({limits.max_rows} 行或约 {max(1, limits.max_bytes // 1_000_000)} MB),只保留了前 " + f"{len(rows)} 行,下游拿到的不是完整结果。请在合并 SQL 中聚合或加条件缩小范围")}) + return out + + +def _duplicates(alias: str, snapshot: dict[str, Any], cols: list[str]) -> str | None: + """这个输入按合并键有没有重复:有的话举第一组,说一共几组。""" + columns = [str(c) for c in snapshot.get("columns") or []] + if not cols or any(c not in columns for c in cols): + return None + at = [columns.index(c) for c in cols] + counts = Counter(tuple(_hashable([row[j]]) for j in at) for row in snapshot.get("rows") or [] + if isinstance(row, list)) + repeated = [(key, n) for key, n in counts.items() if n > 1] + if not repeated: + return None + key, n = repeated[0] + shown = " / ".join(str(json.loads(v)[0]) for v in key) + more = f"等 {len(repeated)} 组" if len(repeated) > 1 else "" + return f"「{alias}」中按({'、'.join(cols)})有重复的键,例如 {shown} 出现 {n} 次{more}" diff --git a/backend/app/engine/nodes/merge.py b/backend/app/engine/nodes/merge.py new file mode 100644 index 00000000..fc668689 --- /dev/null +++ b/backend/app/engine/nodes/merge.py @@ -0,0 +1,132 @@ +"""合并查询节点:读上游的查询快照,在库外按键合并,结果存成和数据源查询同形的查询快照。 + +**读的是工件库里那份完整结果。** 上游 tool 节点交回的 JSON 里带着查询快照的工件 id,这里按 id 取回快照(取回时复验 +哈希),不读交给模型的那份文字:Agent 的消息历史里结果可能被截短,拿截短的文字合并会悄悄少行。上游是另一个合并 +节点时同理,取它的结果快照。 + +**结果和数据源查询同形。** 快照有 columns、rows、truncated、sql、source(记为「合并查询」),台账里记一条 kind 为 +query 的条目:报告照样按先后编成 Qn,[[v:Qn.rK.列]]、[[table:Qn …]] 和口径卡的 cell(nodes.<本节点>, 行, '列') 都按 +原来的规则取值、核对。快照另记 inputs(别名、节点、各自的快照)和合并 SQL,证据面板据此展示合并的来源;逐格来历 +记在 lineage 里(见 engine/merge_query.py),下钻据此指回输入的那一格。 + +**重放结果一致。** 快照里没有耗时这类每次不同的字段:节点重放、继续运行时同样的输入得到同一个工件 id、同一条 +台账,报告里已经写下的出处不会因此变成另一份快照。 +""" +from __future__ import annotations + +import asyncio +import json +import time +from typing import Any + +from app.core import artifact_store +from app.core.events import EventType +from app.data.guard import QueryLimits +from app.engine.context import NodeContext, NodeError +from app.engine.evidence import ledger_enabled, next_exec +from app.engine.merge_query import MERGE_SOURCE, MergeError, MergeInput, execute +from app.engine.state import GraphState + +#: 合并警告进运行事件时的 code(log 事件)。运行详情、运行概况按它认出这是合并查询的警告 +LOG_CODES = {"key_type_mismatch": "merge_key_type", "rows_grew": "merge_rows_grew", + "result_truncated": "merge_truncated"} +#: merge.end 事件里带几行预览:运行详情只是让人看一眼合并出了什么,完整结果在查询快照里 +PREVIEW_ROWS = 5 +#: 上游查询没查成时,tool 节点交回的原话的开头(engine/nodes/tools.py 的 QUERY_FAILED 同一组) +_FAILED_PREFIXES = ("查询失败", "SQL 被拒绝:") + + +def _input_payload(value: Any) -> dict[str, Any] | None: + """上游节点的产出 → 带工件 id 的查询结果(tool 节点交回的是 JSON 文本,合并节点交回的是对象)。""" + data = value + if isinstance(data, str): + try: + data = json.loads(data) + except ValueError: + return None + return data if isinstance(data, dict) else None + + +async def _load_input(state: GraphState, ctx: NodeContext, alias: str, node_id: Any) -> MergeInput: + nodes = ctx.run.spec.node_map() + target = nodes.get(node_id) if isinstance(node_id, str) else None + if target is None: + raise NodeError(ctx.node.id, f"输入「{alias}」选择的节点「{node_id}」不存在,请在节点中重新选择") + who = f"输入「{alias}」({target.title})" + value = (state.get("nodes") or {}).get(target.id) + if value is None: + raise NodeError(ctx.node.id, f"{who}还没有运行结果:请确认它连在合并查询之前") + if isinstance(value, dict) and value.get("skipped"): + raise NodeError(ctx.node.id, f"{who}满足跳过条件被跳过了,没有查询结果可以合并") + if isinstance(value, dict) and value.get("failed"): + raise NodeError(ctx.node.id, f"{who}执行失败,没有查询结果可以合并:{value.get('error') or '原因未知'}") + if isinstance(value, str) and value.startswith(_FAILED_PREFIXES): + raise NodeError(ctx.node.id, f"{who}的查询没有成功,无法合并:{value[:300]}") + payload = _input_payload(value) + artifact = payload.get("artifact") if payload else None + if not isinstance(artifact, str) or not artifact: + raise NodeError(ctx.node.id, f"{who}的结果不是存成快照的查询结果。合并查询只能合并数据库查询(调用数据库查询工具的" + "「调用工具」节点)或另一个「合并查询」节点的结果") + try: + snapshot = await asyncio.to_thread(artifact_store.load, artifact) + except ValueError: + raise NodeError(ctx.node.id, f"{who}的查询快照与哈希不一致,疑似被修改,不能合并") from None + if snapshot is None: + raise NodeError(ctx.node.id, f"{who}的查询快照已不存在,无法合并。请重新运行它") + return MergeInput(alias=alias, node_id=target.id, artifact=artifact, snapshot=snapshot, label=target.title) + + +async def run_merge(state: GraphState, ctx: NodeContext) -> dict[str, Any]: + inputs = ctx.config.get("inputs") + if not isinstance(inputs, dict) or not inputs: + raise NodeError(ctx.node.id, "「合并查询」还没有配置输入:至少选择一个上游的查询节点,并给它起一个别名") + # SQL 里允许 {{ }}(例如按运行输入筛日期),和 tool 节点的 SQL 一样先渲染;快照里存的是渲染之后、守卫规范过的那条 + sql = ctx.render_str(str(ctx.config.get("sql") or ""), state) + loaded = [await _load_input(state, ctx, str(alias), node_id) for alias, node_id in inputs.items()] + + started = time.perf_counter() + try: + # 内存 SQLite 是同步的:放进线程,大一点的合并不卡住事件循环(别的节点的事件照常流出去) + outcome = await asyncio.to_thread(execute, loaded, sql, limits=QueryLimits()) + except MergeError as e: + raise NodeError(ctx.node.id, str(e)) from None + elapsed = int((time.perf_counter() - started) * 1000) + + snapshot = outcome.snapshot() + try: + artifact = await artifact_store.put_json(snapshot, kind="query_snapshot", run_id=ctx.run.run_id, + node_id=ctx.node.id) + except Exception as e: # noqa: BLE001 - 存不下就没有出处可言,不能带着一份引用不了的结果往下走 + raise NodeError(ctx.node.id, "合并结果没能存成查询快照(工件库写入失败),下游无法引用它。" + "请稍后点「继续运行」重试", retryable=True) from e + + labels = {inp.alias: inp.label for inp in loaded} + inputs_view = [{**{k: v for k, v in item.items() if k != "columns"}, "label": labels.get(item["alias"]) or None} + for item in outcome.inputs] + ctx.emit(EventType.MERGE_END, inputs=inputs_view, sql=outcome.sql, rows=len(outcome.rows), + columns=outcome.columns, truncated=outcome.truncated, warnings=outcome.warnings, + preview_rows=outcome.rows[:PREVIEW_ROWS], duration_ms=elapsed, query_artifact=artifact, + lineage=outcome.lineage is not None) + for warning in outcome.warnings: + ctx.emit(EventType.LOG, level="warn", message=warning["message"], + code=LOG_CODES.get(warning["code"], "merge_warning")) + + output: dict[str, Any] = { + "artifact": artifact, "columns": outcome.columns, "rows": outcome.rows, "row_count": len(outcome.rows), + "truncated": outcome.truncated, "sql": outcome.sql, "source": MERGE_SOURCE, "inputs": inputs_view, + "warnings": outcome.warnings, + } + if outcome.column_types: + output["column_types"] = outcome.column_types + updates: dict[str, Any] = {"nodes": {ctx.node.id: output}} + if ledger_enabled(ctx.run): + # 证据台账:和数据源查询同形的 query 条目,报告据此编 Qn;没有 via(外面不包工具快照) + updates["evidence"] = [{ + "kind": "query", "node_id": ctx.node.id, "exec": next_exec(state, ctx.node.id), "artifact": artifact, + "source": MERGE_SOURCE, "columns": outcome.columns, "rows": len(outcome.rows), + "truncated": outcome.truncated, + }] + var_name = str(ctx.config.get("assign_to") or "").strip() + if var_name: + updates["vars"] = {var_name: output} + return updates diff --git a/backend/app/engine/nodes/metrics.py b/backend/app/engine/nodes/metrics.py index b926b0e7..6cb523ea 100644 --- a/backend/app/engine/nodes/metrics.py +++ b/backend/app/engine/nodes/metrics.py @@ -330,7 +330,7 @@ _REF_HEAD = re.compile(r"^(vars|nodes|input)\.([A-Za-z_一-鿿][\w\-一-鿿]*)(. _VIA = {"input": "input", "code": "code", "agent": "agent", "llm": "llm", "tool": "tool", "transform": "transform", "metrics": "metric", "retrieve": "retrieval", "loop": "loop", "human": "human", "validate": "validate", "supervisor": "team", "subgraph": "subgraph", - "memory": "memory"} + "memory": "memory", "merge": "merge"} def _input_of(path: str, value: Any, state: GraphState, ctx: NodeContext, diff --git a/backend/app/engine/schema.py b/backend/app/engine/schema.py index a0be2442..7ed49838 100644 --- a/backend/app/engine/schema.py +++ b/backend/app/engine/schema.py @@ -23,6 +23,7 @@ class NodeType(StrEnum): TOOL = "tool" # 直接调用一个工具 CODE = "code" # 沙箱里跑代码 + MERGE = "merge" # 合并查询:几次查询的完整结果在库外按键合并(engine/merge_query.py) BRANCH = "branch" # 条件分支 LOOP = "loop" # 循环 / 迭代 diff --git a/backend/app/engine/upgrade.py b/backend/app/engine/upgrade.py index 022e39a5..2bb86478 100644 --- a/backend/app/engine/upgrade.py +++ b/backend/app/engine/upgrade.py @@ -53,7 +53,7 @@ _LLM_ONLY = ("prompt", "system", "skills", "use_history", "output_schema") _MODEL_KEYS = ("provider", "model") #: 报告的指令里照搬了这几类节点的文字,就要提醒:报告撰写自己会收集它们的证据 _EVIDENCE_TYPES = (NodeType.METRICS, NodeType.TOOL, NodeType.AGENT, NodeType.RETRIEVE, NodeType.CODE, - NodeType.SUBGRAPH) + NodeType.SUBGRAPH, NodeType.MERGE) #: 写整段文字的节点:出具契约的 narrative 拼了其中几个的文字,就说不准该换哪一个 _WRITER_TYPES = (NodeType.LLM, NodeType.AGENT) _QUESTION = re.compile(r"(? bool: """这个节点查库:报告撰写能把它查到的结果编成 Q1、Q2… 引用。只认数据源的查询工具,别的工具的 - 返回进不了目录。""" + 返回进不了目录。合并查询的结果和数据源查询同形,同样编号。""" + if node.type == NodeType.MERGE: + return True if node.type == NodeType.TOOL: return str(node.config.get("tool") or "").startswith("db_query__") if node.type == NodeType.AGENT: diff --git a/backend/app/engine/variables.py b/backend/app/engine/variables.py index f023f4d3..8f95a20d 100644 --- a/backend/app/engine/variables.py +++ b/backend/app/engine/variables.py @@ -340,6 +340,15 @@ def analyze(spec: GraphSpec) -> VariableReport: node_id=node.id, node_label=label, field=field, expr=m.group(0), )) + # 合并查询的输入按节点 id 选,不写模板:不认这一条的话,变量表里看不出上游那条查询被谁用了。 + # 选错的节点(不存在、不在上游、不是查询)由 validate_graph 报,这里只记引用 + if node.type == "merge" and isinstance(cfg.get("inputs"), dict): + for alias, source in cfg["inputs"].items(): + target = index.get(f"nodes.{source}") if isinstance(source, str) else None + if target is not None: + target.refs.append(VarRef(node_id=node.id, node_label=label, field=f"inputs.{alias}", + expr=f"nodes.{source}")) + for field, text in _iter_strings(cfg): for m in _TEMPLATE_RE.finditer(text): expr = m.group(1).strip() diff --git a/backend/tests/fixtures/merge/members.sql b/backend/tests/fixtures/merge/members.sql new file mode 100644 index 00000000..e0ac8130 --- /dev/null +++ b/backend/tests/fixtures/merge/members.sql @@ -0,0 +1,28 @@ +-- 会员库(虚构):会员到店记录,和门店库是两个独立的库。门店编号同样是文本,但列名不同(store_code)。 +-- 合并查询的端到端测试按「日期 + 门店」聚合到店人数,再和门店库的订单按键合并。数据只为测试编造。 +CREATE TABLE visits ( + visit_id INTEGER PRIMARY KEY, + member_id INTEGER NOT NULL, + store_code TEXT NOT NULL, + visit_date TEXT NOT NULL +); +INSERT INTO visits VALUES + (1, 101, 'S01', '2026-05-01'), + (2, 102, 'S01', '2026-05-01'), + (3, 103, 'S01', '2026-05-01'), + (4, 104, 'S01', '2026-05-01'), + (5, 105, 'S01', '2026-05-01'), + (6, 106, 'S01', '2026-05-01'), + (7, 201, 'S02', '2026-05-01'), + (8, 202, 'S02', '2026-05-01'), + (9, 203, 'S02', '2026-05-01'), + (10, 204, 'S02', '2026-05-01'), + (11, 101, 'S01', '2026-05-02'), + (12, 107, 'S01', '2026-05-02'), + (13, 108, 'S01', '2026-05-02'), + (14, 109, 'S01', '2026-05-02'), + (15, 110, 'S01', '2026-05-02'), + (16, 201, 'S02', '2026-05-02'), + (17, 205, 'S02', '2026-05-02'), + (18, 206, 'S02', '2026-05-02'), + (19, 207, 'S02', '2026-05-02'); diff --git a/backend/tests/fixtures/merge/stores.sql b/backend/tests/fixtures/merge/stores.sql new file mode 100644 index 00000000..ad097469 --- /dev/null +++ b/backend/tests/fixtures/merge/stores.sql @@ -0,0 +1,22 @@ +-- 门店库(虚构):门店和销售订单。合并查询的端到端测试按「日期 + 门店」聚合订单数和销售额。 +-- 门店编号是文本('S01'),金额是实数。数据只为测试编造,和任何真实门店无关。 +CREATE TABLE stores ( + store_id TEXT PRIMARY KEY, + name TEXT NOT NULL +); +CREATE TABLE orders ( + order_id INTEGER PRIMARY KEY, + store_id TEXT NOT NULL REFERENCES stores (store_id), + order_date TEXT NOT NULL, + amount REAL NOT NULL +); +INSERT INTO stores VALUES ('S01', '湖滨店'), ('S02', '东站店'); +INSERT INTO orders VALUES + (1, 'S01', '2026-05-01', 120.0), + (2, 'S01', '2026-05-01', 80.0), + (3, 'S01', '2026-05-01', 100.0), + (4, 'S02', '2026-05-01', 60.0), + (5, 'S02', '2026-05-01', 90.0), + (6, 'S01', '2026-05-02', 200.0), + (7, 'S01', '2026-05-02', 50.0), + (8, 'S02', '2026-05-02', 75.0); diff --git a/backend/tests/test_merge_engine.py b/backend/tests/test_merge_engine.py new file mode 100644 index 00000000..55bda2be --- /dev/null +++ b/backend/tests/test_merge_engine.py @@ -0,0 +1,385 @@ +"""合并查询的库外合并(engine/merge_query.py):几次查询的完整快照落进内存 SQLite,再按一条 SELECT / WITH 合并。 + +纯函数,不连库、不读工件:快照在测试里现造。端到端(真运行、证据面板、推断来源)见 test_merge_node.py。 + +场景是虚构的门店经营:门店库按「日期 + 门店」聚合的销售,会员库按同样粒度聚合的到店人数。 +""" +from __future__ import annotations + +import pytest + +from app.data.guard import QueryLimits +from app.engine.merge_query import ( + MERGE_SOURCE, + MergeError, + MergeInput, + alias_problem, + execute, + sql_problem, + trace_cell, +) +from app.engine.expressions import same_value + +SALES_COLS = ["日期", "门店", "订单数", "销售额"] +SALES = [ + ["2026-05-01", "S01", 40, "3200.50"], + ["2026-05-01", "S02", 25, "1875.00"], + ["2026-05-02", "S01", 38, "3010.00"], + ["2026-05-02", "S02", 30, "2400.00"], +] +VISITS_COLS = ["日期", "门店", "到店人数"] +VISITS = [ + ["2026-05-01", "S01", 160], + ["2026-05-01", "S02", 90], + ["2026-05-02", "S02", 100], + ["2026-05-02", "S01", 150], +] +JOIN_SQL = ("SELECT s.日期, s.门店, s.订单数, s.销售额, v.到店人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 " + "ORDER BY s.日期, s.门店") + + +def snap(columns, rows, *, types=None, truncated=False, source="stores", **extra): + return {"columns": list(columns), "rows": [list(r) for r in rows], "row_count": len(rows), "truncated": truncated, + "sql": "SELECT …", "source": source, **({"column_types": types} if types is not None else {}), **extra} + + +def sales(**kw): + return MergeInput(alias="s", node_id="q_sales", artifact="a" * 64, + snapshot=snap(SALES_COLS, SALES, types={"日期": "text", "门店": "text", "订单数": "number", + "销售额": "number"}, **kw)) + + +def visits(rows=VISITS, *, types=None, **kw): + return MergeInput(alias="v", node_id="q_visits", artifact="b" * 64, + snapshot=snap(VISITS_COLS, rows, types=types or {"日期": "text", "门店": "text", "到店人数": "number"}, + source="members", **kw)) + + +# -------------------------------------------------------------------------- +# 合并本身 +# -------------------------------------------------------------------------- + + +def test_join_on_date_and_store(): + out = execute([sales(), visits()], JOIN_SQL) + assert out.columns == ["日期", "门店", "订单数", "销售额", "到店人数"] + assert out.rows == [ + ["2026-05-01", "S01", 40, 3200.5, 160], + ["2026-05-01", "S02", 25, 1875, 90], + ["2026-05-02", "S01", 38, 3010, 150], + ["2026-05-02", "S02", 30, 2400, 100], + ] + # 数值列按数存:数据源把 DECIMAL 落成的文本("3200.50")合并之后能直接参与运算 + assert out.column_types == {"日期": "text", "门店": "text", "订单数": "number", "销售额": "number", + "到店人数": "number"} + assert out.truncated is False and out.warnings == [] + assert out.sql == JOIN_SQL + + +def test_with_is_allowed_and_text_keeps_leading_zeros(): + a = MergeInput(alias="a", node_id="n1", artifact="c" * 64, + snapshot=snap(["会员号", "积分"], [["007", 10], ["070", 20]], types={"会员号": "text", "积分": "number"})) + out = execute([a], "WITH t AS (SELECT 会员号, 积分 FROM a) SELECT 会员号, 积分 * 2 AS 双倍 FROM t ORDER BY 会员号") + assert out.rows == [["007", 20], ["070", 40]] + + +def test_column_names_are_checked_not_treated_as_strings(): + """内存库同样关掉「双引号当字符串」:写错的列名要报错,不能静默得出一列常量。""" + with pytest.raises(MergeError, match="no such column"): + execute([sales()], 'SELECT "销售颔" FROM s') + + +@pytest.mark.parametrize("sql", [ + "DELETE FROM s", + "UPDATE s SET 订单数 = 0", + "INSERT INTO s VALUES (1, 2, 3, 4)", + "DROP TABLE s", + "PRAGMA table_info(s)", + "ATTACH DATABASE '/tmp/x.db' AS x", + "WITH k AS (SELECT 1) DELETE FROM s", + "EXPLAIN SELECT * FROM s", +]) +def test_only_select_and_with(sql): + assert sql_problem(sql) is not None + with pytest.raises(MergeError): + execute([sales()], sql) + + +def test_one_statement_only(): + problem = sql_problem("SELECT * FROM s; SELECT * FROM v") + assert problem is not None and "一条" in problem + with pytest.raises(MergeError): + execute([sales(), visits()], "SELECT * FROM s; SELECT * FROM v") + + +def test_empty_sql(): + assert "合并 SQL" in sql_problem(" ") + + +def test_static_check_tolerates_templates(): + assert sql_problem("SELECT * FROM s WHERE 日期 >= {{ input.start | json }}") is None + + +@pytest.mark.parametrize("alias, ok", [ + ("s", True), ("sales_2026", True), ("_t", True), + ("", False), ("1s", False), ("a-b", False), ("a b", False), ("订单", False), + ("select", False), ("ORDER", False), ("join", False), ("sqlite_stat", False), ("x" * 40, False), +]) +def test_alias_must_be_a_plain_table_name(alias, ok): + assert (alias_problem(alias) is None) is ok + + +def test_aliases_differing_only_in_case_collide(): + a = MergeInput(alias="t", node_id="n1", artifact="c" * 64, snapshot=snap(["x"], [[1]])) + b = MergeInput(alias="T", node_id="n2", artifact="d" * 64, snapshot=snap(["x"], [[2]])) + with pytest.raises(MergeError, match="别名"): + execute([a, b], "SELECT * FROM t") + + +def test_duplicate_column_names_in_an_input_are_refused_with_a_fix(): + a = MergeInput(alias="t", node_id="n1", artifact="c" * 64, snapshot=snap(["金额", "金额"], [[1, 2]])) + with pytest.raises(MergeError, match="重名"): + execute([a], "SELECT * FROM t") + + +# -------------------------------------------------------------------------- +# 三道防护 +# -------------------------------------------------------------------------- + + +def test_a_truncated_input_is_refused(): + """被截断的输入只有前若干行:拿它按键合并,缺的行不会报错,只会悄悄少算。""" + with pytest.raises(MergeError) as e: + execute([sales(), visits(truncated=True)], JOIN_SQL) + message = str(e.value) + assert "「v」" in message and "截断" in message + assert "聚合" in message and "缩小范围" in message + + +def test_key_type_mismatch_is_warned(): + """门店编号一边是文本 '001',一边是数 1:SQLite 比较时会隐式转换,'001'、'01' 都等于 1。""" + stores = MergeInput(alias="s", node_id="q_sales", artifact="a" * 64, + snapshot=snap(["门店", "销售额"], [["001", 100], ["002", 200]], + types={"门店": "text", "销售额": "number"})) + traffic = MergeInput(alias="v", node_id="q_visits", artifact="b" * 64, + snapshot=snap(["门店", "到店人数"], [[1, 10], [2, 20]], types={"门店": "number", "到店人数": "number"}, + source="members")) + out = execute([stores, traffic], "SELECT s.门店, s.销售额, v.到店人数 FROM s JOIN v ON s.门店 = v.门店") + [warning] = out.warnings + assert warning["code"] == "key_type_mismatch" + assert "s.门店" in warning["message"] and "v.门店" in warning["message"] + assert "'001'" in warning["message"] and "CAST" in warning["message"] + + +def test_key_type_mismatch_is_found_through_using(): + stores = MergeInput(alias="s", node_id="n1", artifact="a" * 64, + snapshot=snap(["门店", "销售额"], [["1", 100]], types={"门店": "text", "销售额": "number"})) + traffic = MergeInput(alias="v", node_id="n2", artifact="b" * 64, + snapshot=snap(["门店", "到店人数"], [[1, 10]], types={"门店": "number", "到店人数": "number"})) + out = execute([stores, traffic], "SELECT 门店, 销售额, 到店人数 FROM s JOIN v USING (门店)") + assert [w["code"] for w in out.warnings] == ["key_type_mismatch"] + + +def test_matching_key_types_raise_no_warning(): + out = execute([sales(), visits()], JOIN_SQL) + assert out.warnings == [] + + +def test_more_rows_than_the_largest_input_is_warned(): + """两边都有一天同一门店出现两行(没聚合到同一粒度):多对多匹配,合并后行数被放大。""" + stores = sales() + stores.snapshot["rows"].append(["2026-05-01", "S01", 2, "100.00"]) + doubled = [*VISITS, ["2026-05-01", "S01", 5]] + out = execute([stores, visits(doubled)], JOIN_SQL) + assert len(out.rows) == 7 + grew = [w for w in out.warnings if w["code"] == "rows_grew"] + assert len(grew) == 1 + message = grew[0]["message"] + assert "7 行" in message and "5 行" in message and "不唯一" in message + # 合并键认得出来时,说出是哪个输入、哪组键重复 + assert "「v」" in message and "2026-05-01" in message and "S01" in message + + +def test_one_to_many_is_not_warned_as_growth(): + """一边的键唯一时,结果行数不会超过另一边:不报。""" + doubled = [*VISITS, ["2026-05-01", "S01", 5]] + out = execute([sales(), visits(doubled)], JOIN_SQL) + assert len(out.rows) == 5 and out.warnings == [] + + +def test_result_over_the_row_limit_is_truncated_and_warned(): + out = execute([sales(), visits()], "SELECT * FROM s CROSS JOIN v", limits=QueryLimits(max_rows=5)) + assert out.truncated is True and len(out.rows) == 5 + assert "result_truncated" in [w["code"] for w in out.warnings] + + +def test_byte_limit_matches_the_query_layer(): + out = execute([sales(), visits()], "SELECT * FROM s CROSS JOIN v", limits=QueryLimits(max_bytes=100)) + assert out.truncated is True and 0 < len(out.rows) < 16 + + +def test_runaway_query_is_stopped(): + endless = "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c) SELECT count(*) FROM c" + with pytest.raises(MergeError, match="秒"): + execute([sales()], endless, limits=QueryLimits(timeout_seconds=0.2)) + + +# -------------------------------------------------------------------------- +# 逐格来历:宁可不下钻,也不下钻错 +# -------------------------------------------------------------------------- + + +def _assert_traces_are_true(inputs, out): + """凡是给出来历的格,都和它指向的那个输入的那一格是同一个值。""" + by_alias = {i.alias: i for i in inputs} + found = 0 + for r, row in enumerate(out.rows): + for column in out.columns: + hit = trace_cell(out.snapshot(), r, column) + if hit is None: + continue + found += 1 + source = by_alias[hit.alias].snapshot + value = source["rows"][hit.row][source["columns"].index(hit.column)] + assert same_value(row[out.columns.index(column)], value), (r, column, hit) + return found + + +def test_direct_columns_of_a_join_trace_to_the_input_rows(): + inputs = [sales(), visits()] + out = execute(inputs, JOIN_SQL) + assert out.lineage is not None + # 2026-05-02 / S01 在会员库里是第 3 行(行号从 0 数),在门店库里是第 2 行 + hit = trace_cell(out.snapshot(), 2, "到店人数") + assert (hit.alias, hit.row, hit.column) == ("v", 3, "到店人数") + hit = trace_cell(out.snapshot(), 2, "销售额") + assert (hit.alias, hit.row, hit.column) == ("s", 2, "销售额") + assert _assert_traces_are_true(inputs, out) == 20 + + +def test_renamed_columns_still_trace(): + inputs = [sales(), visits()] + out = execute(inputs, "SELECT s.门店 AS 店, v.到店人数 人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 " + "WHERE s.日期 = '2026-05-02' ORDER BY 1") + hit = trace_cell(out.snapshot(), 1, "人数") + assert (hit.alias, hit.row, hit.column) == ("v", 2, "到店人数") + assert _assert_traces_are_true(inputs, out) == 4 + + +def test_computed_columns_have_no_trace(): + inputs = [sales(), visits()] + out = execute(inputs, "SELECT s.门店, s.订单数 * 1.0 / v.到店人数 AS 转化率 FROM s JOIN v " + "ON s.日期 = v.日期 AND s.门店 = v.门店 ORDER BY s.日期, s.门店") + assert trace_cell(out.snapshot(), 0, "转化率") is None + assert trace_cell(out.snapshot(), 0, "门店").alias == "s" + + +@pytest.mark.parametrize("sql", [ + "SELECT 日期, sum(订单数) AS 订单数 FROM s GROUP BY 日期", + "SELECT DISTINCT 门店 FROM s", + "SELECT 门店 FROM s UNION SELECT 门店 FROM v", + "SELECT 门店, (SELECT max(到店人数) FROM v) AS 峰值 FROM s", + "SELECT max(订单数) AS 订单数 FROM s", + "WITH t AS (SELECT * FROM s) SELECT 订单数 FROM t", + "SELECT 门店, 订单数, row_number() OVER (ORDER BY 订单数) AS 名次 FROM s", + "SELECT * FROM s NATURAL JOIN v", +]) +def test_shapes_outside_the_grammar_have_no_trace_at_all(sql): + out = execute([sales(), visits()], sql) + assert out.lineage is None + assert all(trace_cell(out.snapshot(), r, c) is None for r in range(len(out.rows)) for c in out.columns) + + +def test_left_join_misses_have_no_trace_on_the_missing_side(): + short = [["2026-05-01", "S01", 160]] + inputs = [sales(), visits(short)] + out = execute(inputs, "SELECT s.日期, s.门店, v.到店人数 FROM s LEFT JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 " + "ORDER BY s.日期, s.门店") + assert out.rows[1][2] is None + assert trace_cell(out.snapshot(), 1, "到店人数") is None + assert trace_cell(out.snapshot(), 1, "门店").alias == "s" + assert trace_cell(out.snapshot(), 0, "到店人数").row == 0 + _assert_traces_are_true(inputs, out) + + +def test_star_expands_in_from_order(): + inputs = [sales(), visits()] + out = execute(inputs, "SELECT * FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 ORDER BY s.日期, s.门店") + assert out.columns == [*SALES_COLS, *VISITS_COLS] + # 重名的列按第一次出现引用:「日期」指的是门店库那一列 + assert trace_cell(out.snapshot(), 0, "日期").alias == "s" + assert trace_cell(out.snapshot(), 0, "到店人数").alias == "v" + _assert_traces_are_true(inputs, out) + + +def test_using_columns_written_bare_have_no_trace(): + """USING 合并掉的列不写表名时,SQLite 取的是哪一边说不准(RIGHT / FULL JOIN 时是两边合起来的值)。""" + out = execute([sales(), visits()], "SELECT 日期, 门店, 订单数, 到店人数 FROM s JOIN v USING (日期, 门店) ORDER BY 1, 2") + assert trace_cell(out.snapshot(), 0, "日期") is None + assert trace_cell(out.snapshot(), 0, "订单数").alias == "s" + + +def test_identical_output_rows_are_not_traced(): + """两行结果一模一样时,说不清哪一行来自哪一行输入:不给。""" + twins = [["2026-05-01", "S01", 7], ["2026-05-01", "S01", 7], ["2026-05-02", "S02", 9]] + inputs = [MergeInput(alias="v", node_id="n", artifact="b" * 64, snapshot=snap(VISITS_COLS, twins))] + out = execute(inputs, "SELECT 日期, 门店, 到店人数 FROM v") + assert trace_cell(out.snapshot(), 0, "到店人数") is None and trace_cell(out.snapshot(), 1, "到店人数") is None + assert trace_cell(out.snapshot(), 2, "到店人数").row == 2 + + +def test_self_join_traces_each_side_separately(): + inputs = [visits()] + out = execute(inputs, "SELECT a.门店, a.到店人数 AS 首日, b.到店人数 AS 次日 FROM v AS a JOIN v AS b " + "ON a.门店 = b.门店 AND a.日期 = '2026-05-01' AND b.日期 = '2026-05-02' ORDER BY a.门店") + assert out.rows == [["S01", 160, 150], ["S02", 90, 100]] + first, second = trace_cell(out.snapshot(), 0, "首日"), trace_cell(out.snapshot(), 0, "次日") + assert (first.alias, first.row) == ("v", 0) and (second.alias, second.row) == ("v", 3) + _assert_traces_are_true(inputs, out) + + +def test_a_rowid_column_in_the_input_does_not_confuse_the_trace(): + rows = [[9, "S01", 1], [8, "S02", 2]] + inputs = [MergeInput(alias="t", node_id="n", artifact="b" * 64, snapshot=snap(["rowid", "门店", "值"], rows))] + out = execute(inputs, "SELECT 门店, 值 FROM t ORDER BY 门店 DESC") + assert out.rows == [["S02", 2], ["S01", 1]] + assert trace_cell(out.snapshot(), 0, "值").row == 1 + _assert_traces_are_true(inputs, out) + + +# -------------------------------------------------------------------------- +# 快照形状 +# -------------------------------------------------------------------------- + + +def test_snapshot_has_the_query_shape_plus_inputs(): + out = execute([sales(), visits()], JOIN_SQL) + snapshot = out.snapshot() + for key in ("columns", "rows", "row_count", "truncated", "sql", "column_types"): + assert key in snapshot + assert snapshot["source"] == MERGE_SOURCE + assert snapshot["inputs"] == [ + {"alias": "s", "node_id": "q_sales", "artifact": "a" * 64, "rows": 4, "columns": SALES_COLS, "source": "stores"}, + {"alias": "v", "node_id": "q_visits", "artifact": "b" * 64, "rows": 4, "columns": VISITS_COLS, + "source": "members"}, + ] + assert snapshot["sources"] == ["stores", "members"] + # 内容确定:同样的输入、同样的 SQL,快照逐字相同(重放、继续运行得到同一个工件) + assert execute([sales(), visits()], JOIN_SQL).snapshot() == snapshot + + +def test_masked_input_columns_stay_masked_even_when_renamed(): + members = MergeInput(alias="v", node_id="q_visits", artifact="b" * 64, + snapshot=snap(["门店", "手机号"], [["S01", "138****"]], source="members", + mask_columns=["手机号"])) + out = execute([sales(), members], "SELECT s.门店, v.手机号 AS 联系方式 FROM s JOIN v ON s.门店 = v.门店") + assert set(out.snapshot()["mask_columns"]) == {"手机号", "联系方式"} + + +def test_nested_merges_keep_the_underlying_sources(): + first = execute([sales(), visits()], JOIN_SQL) + nested = MergeInput(alias="m", node_id="merge1", artifact="e" * 64, snapshot=first.snapshot()) + out = execute([nested], "SELECT 日期, 门店, 到店人数 FROM m WHERE 门店 = 'S02'") + assert out.snapshot()["sources"] == ["stores", "members"] + hit = trace_cell(out.snapshot(), 0, "到店人数") + assert (hit.alias, hit.row) == ("m", 1) diff --git a/backend/tests/test_merge_node.py b/backend/tests/test_merge_node.py new file mode 100644 index 00000000..9c1ee4d8 --- /dev/null +++ b/backend/tests/test_merge_node.py @@ -0,0 +1,456 @@ +"""合并查询节点(merge):画图时的校验、执行、证据链和下钻、助手说明。 + +端到端用两个独立的 SQLite 夹具(tests/fixtures/merge/):门店库的销售订单、会员库的到店记录。两条查询各自在自己的 +库里聚合到「日期 + 门店」,合并查询在库外按键合并,口径卡用 cell(nodes.merge, …) 算转化率和客单价,报告引用合并 +结果的单元格和口径卡的指标。断言报告里的数能追到两边的原始查询,下钻不指错格子。 + +不调用任何模型:报告的写作者换成脚本(monkeypatch MockChatModel._decide)。 +""" +from __future__ import annotations + +import asyncio +import json +import sqlite3 +from pathlib import Path + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage +from sqlalchemy import select + +from app.core import artifact_store +from app.core.config import settings +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource, Run, RunEvent +from app.engine.evidence import iter_segments +from app.engine.merge_query import MERGE_SOURCE +from app.engine.runner import run_manager +from app.engine.schema import GraphSpec, NodeType, validate_graph +from app.main import app + +FIXTURES = Path(__file__).parent / "fixtures" / "merge" + +SALES_SQL = ("SELECT order_date AS 日期, store_id AS 门店, COUNT(*) AS 订单数, SUM(amount) AS 销售额 FROM orders " + "GROUP BY order_date, store_id ORDER BY order_date, store_id") +# 故意和门店库排成不同的顺序:合并结果的第 0 行在会员库的结果里是第 2 行 +VISITS_SQL = ("SELECT visit_date AS 日期, store_code AS 门店, COUNT(*) AS 到店人数 FROM visits " + "GROUP BY visit_date, store_code ORDER BY visit_date DESC, store_code") +MERGE_SQL = ("SELECT s.日期, s.门店, s.订单数, s.销售额, v.到店人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 " + "ORDER BY s.日期, s.门店") +TEXT = ("5 月 1 日湖滨店到店 [[v:Q3.r0.到店人数]] 人,转化率 [[m:conversion]],客单价 [[m:aov]]。\n\n" + "[[table:Q3 cols=日期,门店,订单数,到店人数 rows=0-3]]") + + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def _source(name: str, path: Path, options: dict | None = None) -> str: + async with SessionLocal() as session: + row = (await session.execute(select(DataSource).where(DataSource.name == name))).scalar_one_or_none() + if row is None: + row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + row.database, row.kind, row.enabled, row.readonly, row.options = str(path), "sqlite", True, True, options or {} + await session.commit() + return row.id + + +@pytest.fixture +async def sources(tmp_path, engine_up): + """把两个夹具脚本建成两个独立的 SQLite 库,各接成一个只读数据源。""" + ids = [] + for name in ("stores", "members"): + path = tmp_path / f"{name}.db" + db = sqlite3.connect(path) + db.executescript((FIXTURES / f"{name}.sql").read_text("utf-8")) + db.commit() + db.close() + ids.append(await _source(f"merge_{name}", path)) + for sid in ids: + await engines.invalidate(sid) + yield + for sid in ids: + await engines.invalidate(sid) + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def edge(a, b): + return {"source": a, "target": b} + + +def store_graph(*, sales_sql=SALES_SQL, visits_sql=VISITS_SQL, merge_sql=MERGE_SQL, visits_args=None, report=True, + inputs=None): + """入口分两路查两个库,汇合到合并查询,再接口径卡、报告、成果。""" + nodes = [ + node("start", "input"), + node("q_sales", "tool", tool="db_query__merge_stores", args={"sql": sales_sql}), + node("q_visits", "tool", tool="db_query__merge_members", args={"sql": visits_sql, **(visits_args or {})}), + node("merge", "merge", inputs=inputs or {"s": "q_sales", "v": "q_visits"}, sql=merge_sql), + node("card", "metrics", caliber="门店经营", caliber_version="v1", metrics=[ + {"id": "conversion", "name": "转化率", "decimals": 4, "format": "percent_of_ratio", + "expression": "cell(nodes.merge, 0, '订单数') / cell(nodes.merge, 0, '到店人数')"}, + {"id": "aov", "name": "客单价", "unit": "元", "decimals": 1, + "expression": "cell(nodes.merge, 0, '销售额') / cell(nodes.merge, 0, '订单数')"}]), + ] + edges = [edge("start", "q_sales"), edge("start", "q_visits"), edge("q_sales", "merge"), edge("q_visits", "merge"), + edge("merge", "card")] + if report: + nodes += [node("write", "report", instructions="写门店日报", on_violation="flag"), + node("out", "output", fields=[{"name": "日报", "value": "{{ nodes.write.text }}"}])] + edges += [edge("card", "write"), edge("write", "out")] + else: + nodes.append(node("out", "output", fields=[{"name": "转化率", "value": "{{ nodes.card.metrics }}"}])) + edges.append(edge("card", "out")) + return {"nodes": nodes, "edges": edges} + + +def script(monkeypatch, text=TEXT): + from app.providers import mock_model + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text)) + + +async def wait(run_id: str) -> Run: + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + return row + raise AssertionError(f"run {run_id} 没有结束:{row.status}") + + +async def run_graph(monkeypatch, graph=None, text=TEXT) -> Run: + script(monkeypatch, text) + return await wait((await run_manager.start(graph=graph or store_graph(), input_payload={})).id) + + +async def events(run_id: str, etype: str, node_id: str | None = None) -> list[dict]: + async with SessionLocal() as session: + query = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype) + if node_id: + query = query.where(RunEvent.node_id == node_id) + return [e.data for e in (await session.execute(query.order_by(RunEvent.seq))).scalars()] + + +def load_doc(row: Run) -> dict: + doc_id = row.output["_evidence"]["doc_artifact"] + return json.loads((settings.data_dir / "artifacts" / doc_id[:2] / f"{doc_id}.json").read_text("utf-8")) + + +def seg_of(doc: dict, text: str, n: int = 0) -> dict: + return [s for s in iter_segments(doc) if s["text"] == text and s.get("cite")][n] + + +def alias_of(doc: dict, node_id: str) -> str: + [alias] = [a for a, e in doc["catalog"].items() if e.get("kind") == "query" and e.get("node_id") == node_id] + return alias + + +# -------------------------------------------------------------------------- +# 画图时的校验 +# -------------------------------------------------------------------------- + + +def merge_spec(merge_cfg: dict, *, extra_nodes=(), extra_edges=()) -> GraphSpec: + base = store_graph(report=False) + nodes = [n if n["id"] != "merge" else node("merge", "merge", **merge_cfg) for n in base["nodes"]] + return GraphSpec.model_validate({"nodes": [*nodes, *extra_nodes], "edges": [*base["edges"], *extra_edges]}) + + +def merge_errors(spec: GraphSpec) -> list: + return [i for i in validate_graph(spec).issues if i.node_id == "merge" and i.level == "error"] + + +def test_node_type_exists(): + assert NodeType.MERGE == "merge" + + +def test_a_well_formed_merge_validates(): + spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL}) + assert merge_errors(spec) == [] + + +def test_merge_of_a_merge_is_allowed(): + spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL}, + extra_nodes=[node("again", "merge", inputs={"m": "merge"}, sql="SELECT * FROM m")], + extra_edges=[edge("merge", "again")]) + assert [i for i in validate_graph(spec).issues if i.node_id == "again" and i.level == "error"] == [] + + +def test_inputs_are_required(): + [issue] = merge_errors(merge_spec({"inputs": {}, "sql": MERGE_SQL})) + assert issue.field == "inputs" and issue.code == "merge.inputs_missing" + + +def test_input_pointing_to_a_missing_node(): + [issue] = merge_errors(merge_spec({"inputs": {"s": "q_sales", "v": "nope"}, "sql": MERGE_SQL})) + assert issue.field == "inputs.v" and "不存在" in issue.message and issue.code == "merge.input_invalid" + + +@pytest.mark.parametrize("extra, why", [ + (node("think", "llm", prompt="写点什么"), "不产出查询结果"), + (node("calc", "tool", tool="calculator", args={"expression": "1+1"}), "不是数据库查询工具"), + (node("asker", "agent", prompt="查一下", tools=["db_query__merge_stores"]), "Agent"), +]) +def test_input_must_produce_a_query_result(extra, why): + spec = merge_spec({"inputs": {"s": "q_sales", "v": extra["id"]}, "sql": MERGE_SQL}, + extra_nodes=[extra], extra_edges=[edge("start", extra["id"]), edge(extra["id"], "merge")]) + [issue] = merge_errors(spec) + assert issue.field == "inputs.v" and why in issue.message and issue.code == "merge.input_invalid" + + +def test_agent_inputs_explain_how_to_fix(): + extra = node("asker", "agent", prompt="查一下", tools=["db_query__merge_stores"]) + spec = merge_spec({"inputs": {"s": "q_sales", "v": "asker"}, "sql": MERGE_SQL}, + extra_nodes=[extra], extra_edges=[edge("start", "asker"), edge("asker", "merge")]) + [issue] = merge_errors(spec) + assert "查询多次" in issue.message and "调用工具" in issue.message + + +def test_input_must_be_upstream(): + lonely = node("q_other", "tool", tool="db_query__merge_stores", args={"sql": "SELECT 1 AS x"}) + spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_other"}, "sql": MERGE_SQL}, + extra_nodes=[lonely], extra_edges=[edge("start", "q_other")]) + [issue] = merge_errors(spec) + assert issue.field == "inputs.v" and "上游" in issue.message + + +@pytest.mark.parametrize("alias", ["1s", "a-b", "select", "sqlite_x"]) +def test_alias_must_be_a_legal_table_name(alias): + [issue] = merge_errors(merge_spec({"inputs": {"s": "q_sales", alias: "q_visits"}, "sql": MERGE_SQL})) + assert issue.field == f"inputs.{alias}" and issue.code == "merge.alias_invalid" and alias in issue.message + + +@pytest.mark.parametrize("sql, why", [ + ("", "还没有填写"), + ("DELETE FROM s", "SELECT 或 WITH"), + ("PRAGMA table_info(s)", "SELECT 或 WITH"), + ("SELECT * FROM s; SELECT * FROM v", "一条"), +]) +def test_sql_must_be_one_select_or_with(sql, why): + [issue] = merge_errors(merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": sql})) + assert issue.field == "sql" and why in issue.message and issue.code == "merge.sql_invalid" + + +def test_merge_inputs_count_as_references_to_the_upstream_nodes(): + from app.engine.variables import analyze + + report = analyze(merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL})) + refs = {v.path: v.refs for v in report.variables} + assert any(r.node_id == "merge" for r in refs["nodes.q_sales"]) + + +def test_labels_and_governance_know_the_type(): + from app.engine.governance import _traceable + from app.engine.labels import field_label, type_label + from app.engine.upgrade import _queries + + spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL}) + merge = spec.node_map()["merge"] + assert type_label("merge") == "合并查询" + assert field_label("sql", "merge") == "合并 SQL" and field_label("inputs", "merge") == "输入" + assert _traceable(merge) and _queries(merge) + + +# -------------------------------------------------------------------------- +# 执行与证据链 +# -------------------------------------------------------------------------- + + +async def test_end_to_end_two_databases(monkeypatch, sources, client): + row = await run_graph(monkeypatch) + assert row.status == "succeeded", row.error + + [finished] = await events(row.id, "node.finished", "merge") + output = artifact_store.load(finished["artifact"]) + assert output["columns"] == ["日期", "门店", "订单数", "销售额", "到店人数"] + assert output["rows"] == [["2026-05-01", "S01", 3, 300, 6], ["2026-05-01", "S02", 2, 150, 4], + ["2026-05-02", "S01", 2, 250, 5], ["2026-05-02", "S02", 1, 75, 4]] + assert output["source"] == MERGE_SOURCE and output["warnings"] == [] + + # 台账:一条和数据源查询同形的 query 条目,报告据此编号 + [entry] = finished["evidence"] + assert entry["kind"] == "query" and entry["source"] == MERGE_SOURCE and entry["artifact"] == output["artifact"] + assert entry["rows"] == 4 and entry["truncated"] is False + snapshot = artifact_store.load(entry["artifact"]) + assert [i["alias"] for i in snapshot["inputs"]] == ["s", "v"] + assert snapshot["sources"] == ["merge_stores", "merge_members"] + assert snapshot["sql"] == MERGE_SQL + + # 运行事件:合并的输入、SQL、行数,运行详情据此展示 + [done] = await events(row.id, "merge.end", "merge") + assert done["sql"] == MERGE_SQL and done["rows"] == 4 and done["warnings"] == [] + assert [(i["alias"], i["node_id"], i["rows"], i["source"]) for i in done["inputs"]] == [ + ("s", "q_sales", 4, "merge_stores"), ("v", "q_visits", 4, "merge_members")] + assert done["query_artifact"] == entry["artifact"] + + doc = load_doc(row) + q3 = doc["catalog"]["Q3"] + assert q3["node_id"] == "merge" and q3["source"] == MERGE_SOURCE and q3["artifact"] == entry["artifact"] + assert {alias_of(doc, "q_sales"), alias_of(doc, "q_visits")} == {"Q1", "Q2"} + metric = doc["catalog"]["m:conversion"] + assert metric["value"] == 0.5 and metric["rendered"] == "50.00%" + # 口径卡的两个输入都按 cell() 落在合并结果的那一格上,并按快照核对过 + card = artifact_store.load(metric["artifact"]) + conversion = next(m for m in card["metrics"] if m["id"] == "conversion") + assert [(i["via"], i["status"], i["artifact"], i["locator"]) for i in conversion["inputs"]] == [ + ("tool_cell", "verified", entry["artifact"], {"row": 0, "column": "订单数"}), + ("tool_cell", "verified", entry["artifact"], {"row": 0, "column": "到店人数"})] + text = row.output["日报"] + assert "到店 6 人" in text and "50.00%" in text and "100.0元" in text + + +async def test_cell_segment_traces_to_both_original_queries(monkeypatch, sources, client): + row = await run_graph(monkeypatch) + assert row.status == "succeeded", row.error + doc = load_doc(row) + qv = alias_of(doc, "q_visits") + qs = alias_of(doc, "q_sales") + seg = seg_of(doc, "6") + body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}")).json() + merge_step, visits_step = body["chain"] + + assert merge_step["step"] == "query" and merge_step["alias"] == "Q3" and merge_step["source"] == MERGE_SOURCE + assert merge_step["hash_ok"] is True and merge_step["sealed"] is True + assert "mask_note" not in merge_step # 合并查询不是数据源,不能说它「已不存在」 + merged = merge_step["merge"] + assert merged["sql"] == MERGE_SQL and merged["warnings"] == [] + assert [(i["alias"], i["node_id"], i["query"], i["rows"], i["source"], i["sealed"]) for i in merged["inputs"]] == [ + ("s", "q_sales", qs, 4, "merge_stores", True), ("v", "q_visits", qv, 4, "merge_members", True)] + assert merged["traced"] == [{"cell": [0, "到店人数"], "input": "v", "query": qv, "row": 2, "column": "到店人数"}] + + # 接着给出会员库那次查询,高亮的正是那一格,值相同 + assert visits_step["step"] == "query" and visits_step["alias"] == qv and visits_step["merged_into"] == "Q3" + assert visits_step["highlight"]["cells"] == [[2, "到店人数"]] + at = visits_step["row_index"].index(2) + assert visits_step["rows"][at][visits_step["columns"].index("到店人数")] == 6 + + +async def test_metric_segment_traces_through_the_merge(monkeypatch, sources, client): + row = await run_graph(monkeypatch) + doc = load_doc(row) + qs, qv = alias_of(doc, "q_sales"), alias_of(doc, "q_visits") + body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_of(doc, '50.00%')['id']}")).json() + metric, *inputs = body["chain"][:3] + assert metric["step"] == "metric" and metric["value"] == 0.5 + assert [(i["via"], i["status"], i["query"], i["cell"]) for i in inputs] == [ + ("tool_cell", "verified", "Q3", "Q3.r0.订单数"), ("tool_cell", "verified", "Q3", "Q3.r0.到店人数")] + steps = [s for s in body["chain"] if s["step"] == "query"] + assert [s["alias"] for s in steps] == ["Q3", qs, qv] + merge_step, sales_step, visits_step = steps + assert sorted(t["input"] for t in merge_step["merge"]["traced"]) == ["s", "v"] + assert sales_step["highlight"]["cells"] == [[0, "订单数"]] and sales_step["merged_into"] == "Q3" + assert visits_step["highlight"]["cells"] == [[2, "到店人数"]] + + +async def test_provenance_of_a_merged_cell_follows_the_trace(monkeypatch, sources, client): + """手工源上的格:追到会员库那次查询,结论和直接引用那一格相同(not_upload),并记下经过的合并。""" + row = await run_graph(monkeypatch) + doc = load_doc(row) + seg = seg_of(doc, "6") + out = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}/provenance")).json() + assert (out["status"], out["reason"]["code"]) == ("none", "not_upload") + assert out["cell"] == {"alias": "Q3", "row": 0, "column": "到店人数", "artifact": doc["catalog"]["Q3"]["artifact"]} + assert out["merge"] == [{"alias": "Q3", "node_id": "merge", "input": "v", "query": alias_of(doc, "q_visits"), + "row": 2, "column": "到店人数"}] + + +async def test_a_computed_merge_column_has_no_cell_lineage(monkeypatch, sources, client): + sql = ("SELECT s.日期, s.门店, s.订单数 + v.到店人数 AS 合计 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 " + "ORDER BY s.日期, s.门店") + graph = store_graph(merge_sql=sql) + graph["nodes"] = [n for n in graph["nodes"] if n["id"] != "card"] + graph["edges"] = [e for e in graph["edges"] if "card" not in (e["source"], e["target"])] + [edge("merge", "write")] + row = await run_graph(monkeypatch, graph, text="合计 [[v:Q3.r0.合计]]。") + assert row.status == "succeeded", row.error + doc = load_doc(row) + seg = seg_of(doc, "9") + body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}")).json() + [merge_step] = body["chain"] + [traced] = merge_step["merge"]["traced"] + assert traced["input"] is None and traced["query"] is None and "逐格来历" in traced["note"] + out = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}/provenance")).json() + assert (out["status"], out["reason"]["code"], out["version"]) == ("none", "merge_no_lineage", None) + assert out["merge"] == [{"alias": "Q3", "node_id": "merge", "input": None, "query": None, "row": None, + "column": None}] + + +async def test_a_truncated_input_fails_the_node(monkeypatch, sources): + row = await run_graph(monkeypatch, store_graph(visits_args={"limit": 2}, report=False)) + assert row.status == "failed" + [failed] = await events(row.id, "node.failed", "merge") + assert "「v」" in failed["error"] and "截断" in failed["error"] and "聚合" in failed["error"] + + +async def test_warnings_reach_the_output_and_the_events(monkeypatch, sources): + """门店编号一边是文本 '01'、一边是数 1;只按门店合并、没按日期:两道警告都要看得见。""" + sales_sql = ("SELECT order_date AS 日期, substr(store_id, 2) AS 门店, COUNT(*) AS 订单数, SUM(amount) AS 销售额 " + "FROM orders GROUP BY order_date, store_id ORDER BY 1, 2") + visits_sql = ("SELECT visit_date AS 日期, CAST(substr(store_code, 2) AS INTEGER) AS 门店, COUNT(*) AS 到店人数 " + "FROM visits GROUP BY visit_date, store_code ORDER BY 1, 2") + merge_sql = "SELECT s.门店, s.订单数, s.销售额, v.到店人数 FROM s JOIN v ON s.门店 = v.门店" + row = await run_graph(monkeypatch, store_graph(sales_sql=sales_sql, visits_sql=visits_sql, merge_sql=merge_sql, + report=False)) + assert row.status == "succeeded", row.error + [finished] = await events(row.id, "node.finished", "merge") + output = artifact_store.load(finished["artifact"]) + assert sorted(w["code"] for w in output["warnings"]) == ["key_type_mismatch", "rows_grew"] + [done] = await events(row.id, "merge.end", "merge") + assert sorted(w["code"] for w in done["warnings"]) == ["key_type_mismatch", "rows_grew"] + logs = [e for e in await events(row.id, "log", "merge") if e.get("level") == "warn"] + assert sorted(e["code"] for e in logs) == ["merge_key_type", "merge_rows_grew"] + assert all(e["message"] for e in logs) + + +async def test_rerunning_the_node_on_the_same_inputs_gives_the_same_snapshot(monkeypatch, sources): + """重放、继续运行时节点从头再执行:同样的输入得到同一份快照、同一条台账,下游的出处不变。""" + from app.engine.context import NodeContext, RunContext + from app.engine.nodes.merge import run_merge + + row = await run_graph(monkeypatch, store_graph(report=False)) + assert row.status == "succeeded", row.error + outputs = {} + for nid in ("q_sales", "q_visits"): + [finished] = await events(row.id, "node.finished", nid) + outputs[nid] = artifact_store.load(finished["artifact"]) + spec = GraphSpec.model_validate(store_graph(report=False)) + ctx = NodeContext(node=spec.node_map()["merge"], + run=RunContext(run_id=row.id, thread_id=row.id, spec=spec, agent_limits={})) + state = {"nodes": outputs, "trail": [], "evidence": []} + first, second = await run_merge(state, ctx), await run_merge(state, ctx) + assert first == second + assert first["evidence"][0]["artifact"] == first["nodes"]["merge"]["artifact"] + + +# -------------------------------------------------------------------------- +# 助手 +# -------------------------------------------------------------------------- + + +def test_node_reference_explains_when_to_use_merge(): + from app.api.copilot import NODE_REFERENCE + + assert "- merge:" in NODE_REFERENCE + block = NODE_REFERENCE.split("- merge:", 1)[1].split("\n- ", 1)[0] + for phrase in ("inputs", "sql", "不同的库", "同一个库", "一条 SQL", "口径卡", "截断", "cell(nodes."): + assert phrase in block, phrase