合并查询节点(一):新增 merge 节点和执行器——几次查询的完整快照落进内存 SQLite 按键合并,结果存成同形的查询快照,截断的输入拒绝、键类型不一致和行数放大给警告,并记下逐格来历

This commit is contained in:
ThearchyHelios
2026-10-03 09:27:48 +08:00
parent 67527fcf05
commit 1f8c180703
15 changed files with 1934 additions and 6 deletions
+3
View File
@@ -48,6 +48,9 @@ class EventType(StrEnum):
# 记忆读写。以前发的是 info 日志,而解码器丢弃所有 info——于是往长期记忆里
# 写东西这件事在界面上是隐形的。让 Copilot 主动记之后,这条不可接受
MEMORY_END = "memory.end"
# 合并查询做完了:哪几个输入(别名、节点、行数、数据源)、哪条合并 SQL、得到几行、有哪些警告。
# 运行详情据此展示这一步;完整结果在 query_artifact 指向的查询快照里
MERGE_END = "merge.end"
SANDBOX_START = "sandbox.start"
SANDBOX_END = "sandbox.end"
+9
View File
@@ -277,6 +277,15 @@ def open_checked_sqlite(path: str, *, readonly: bool = True, immutable: bool = F
factory=_DqsOffConnection)
def open_memory_sqlite() -> sqlite3.Connection:
"""一个内存 SQLite 连接,DQS 已关(合并查询把几次查询的结果落进来再合并,engine/merge_query.py)。
合并 SQL 同样是模型写的:双引号里的列名写错一个字,默认连接会把它当成字符串常量,`SUM("销售颔")` 得 0,
`ON s."门店" = v."门店 "` 一行都对不上却不报错。所以和连接层共用 _DqsOffConnection,关不掉就不交出连接。
"""
return sqlite3.connect(":memory:", factory=_DqsOffConnection)
def _sqlite_extra() -> dict[str, Any]:
"""每个 SQLite engine 都带的连接参数:关 DQS。"""
return {"connect_args": {"factory": _DqsOffConnection}}
+2 -1
View File
@@ -11,7 +11,7 @@ from langgraph.graph import END, START, StateGraph
from app.core.errors import describe_exception, raw_detail
from app.core.events import EventType
from app.engine.context import NodeContext, NodeError, RunContext
from app.engine.nodes import control, human, io, knowledge, llm, metrics, multi, report, tools
from app.engine.nodes import control, human, io, knowledge, llm, merge, metrics, multi, report, tools
from app.engine.schema import GraphNode, GraphSpec, NodeType, back_edges, innermost_loops
from app.engine.state import GraphState
@@ -28,6 +28,7 @@ RUNNERS: dict[NodeType, NodeRunner] = {
NodeType.SUBGRAPH: multi.run_subgraph,
NodeType.TOOL: tools.run_tool,
NodeType.CODE: tools.run_code,
NodeType.MERGE: merge.run_merge,
NodeType.BRANCH: control.run_branch,
NodeType.LOOP: control.run_loop,
NodeType.HUMAN: human.run_human,
+1 -1
View File
@@ -237,7 +237,7 @@ REPORT_POLICY_ALSO: dict[str, tuple[str, ...]] = {"claims": (CLAIMS_JUDGE,)}
JUDGE_UNLIMITED = "不设上限,费用只受句数、时长上限和每日上限约束"
#: 产出本身就是证据、或者由系统确定地算出来的节点。成果字段取它们的值,不算「别的节点写的字」
_TRACEABLE = (NodeType.REPORT, NodeType.METRICS, NodeType.INPUT, NodeType.TOOL, NodeType.RETRIEVE)
_TRACEABLE = (NodeType.REPORT, NodeType.METRICS, NodeType.INPUT, NodeType.TOOL, NodeType.RETRIEVE, NodeType.MERGE)
_WRITERS = (NodeType.LLM, NodeType.AGENT, NodeType.SUPERVISOR)
#: 往对话里写消息的节点(emit_message 没关时),「最后一条消息」就是它们里最后跑的那个写的
_EMITTERS = (*_WRITERS, NodeType.REPORT)
+2 -1
View File
@@ -18,7 +18,7 @@ from typing import Any
#: 节点类型的叫法,和前端 lib/terms.ts 的 NODE_TYPE_LABEL 一致
TYPE_LABEL: dict[str, str] = {
"input": "输入", "output": "成果", "llm": "模型调用", "agent": "Agent",
"supervisor": "多 Agent 协作", "tool": "调用工具", "code": "沙箱代码",
"supervisor": "多 Agent 协作", "tool": "调用工具", "code": "沙箱代码", "merge": "合并查询",
"branch": "条件分支", "loop": "循环", "subgraph": "子工作流",
"memory": "长期记忆", "retrieve": "知识检索", "transform": "数据整形",
"human": "人工审批", "validate": "结构校验", "metrics": "口径卡", "report": "报告撰写",
@@ -79,6 +79,7 @@ _FIELD_BY_TYPE: dict[str, dict[str, str]] = {
"human": {"mode": "审批方式", "title": "标题", "message": "展示给审批人的内容", "draft": "草稿内容",
"stop_on_reject": "驳回即终止运行"},
"validate": {"source": "待校验内容", "fail_fast": "校验失败即中断"},
"merge": {"inputs": "输入", "sql": "合并 SQL"},
}
#: 出具契约里的键(canvas/issues.ts 的 CONTRACT_KEYS、SUB_KEYS)
+879
View File
@@ -0,0 +1,879 @@
"""合并查询(merge 节点):几次查询的完整结果落进内存 SQLite,再用一条 SELECT / WITH 按键合并。
**为什么要有它。** 跨查询组合原来只有三种:口径卡用 cell() 组合几个单值;tool 节点的 SQL 里写 {{ vars.x }},把前一条
查询的结果代进后一条;同一个库里写一条 JOIN / WITH。按行合并两次查询,尤其数据在两个库里时,没有路可走:用沙箱代码
合并,出处链断在代码节点上,喂给口径卡时只能提示「核对不了出处」;让模型把 id 从一条查询抄进下一条,会抄错,结果被
截断后还会漏。
**为什么不做跨库联邦查询。** 联邦查询要把两边的明细拉到一处,数据量、方言、权限一样都控制不住。方案是先在各自的库里
聚合到共同的键和粒度(日期 + 门店),再由这里在库外按键合并:两边进来的都是已经聚合过的小结果。
**为什么落成 SQLite、用标准库 sqlite3。** 照搬 data/tabular.py:SQLite 是现成的 SQL 引擎,守卫有它的读法(guard
的 _SQLITE),证据下钻的识别器(direct_select)也按它分词,不必为这一步引入 DuckDB 再多一种方言。连接同样关掉 DQS
(data.engine.open_memory_sqlite):双引号里的列名写错要报错,不能当成字符串常量。
**守卫和三道防护。**
- SQL 先过守卫(SQLite 读法),只认 SELECT 和 WITH;表落完之后连接设成 query_only,再挂授权回调,只放行读表、
调函数、递归公用表表达式。守卫是第一道,后两道不赌守卫没有漏洞;
- 结果的行数、字节上限和查询层相同(QueryLimits),超时按查询层的缺省时限由进度回调中断;
- 任一输入被截断 → 拒绝合并:拿只有前若干行的结果按键合并,缺的行不报错,只会悄悄少算;
- 合并键两边类型不一致(文本 '001' 对数值 1)→ 警告。SQLite 比较时按亲和性隐式转换,'001'、'01' 都会等于 1;
- 结果行数多于行数最多的输入 → 警告合并键可能不唯一(多对多匹配把行放大了)。
**逐格来历:宁可不下钻,也不下钻错。** 报告里引用合并结果的一格,证据面板要能指回它来自哪个输入的哪一格。只有一种
写法认得出来:单层 SELECT,FROM 后面是输入表(可加别名、可 JOIN),选取项是某张输入表的列(可改名);没有分组、
聚合、DISTINCT、子查询、公用表表达式、集合运算、窗口和 NATURAL JOIN。认出来以后,在原 SQL 的选取列表末尾补上每张
输入表的 rowid 再执行一次:补列的结果去掉补的几列,必须和原结果逐行逐值相同,才把 rowid 记成来历;对不上整份不记。
另外两种情况也不给:结果里一模一样的两行(说不清哪一行来自哪一行输入),以及来历那一格和结果的值对不上的列。
表达式、聚合出来的列没有逐格来历,只有表级来历:来自哪几个输入、哪条合并 SQL。
纯函数:不连数据库、不读写工件、不发事件。取上游快照、存快照、发事件在执行器(engine/nodes/merge.py)。
"""
from __future__ import annotations
import json
import re
import sqlite3
import time
from collections import Counter
from dataclasses import dataclass, field
from typing import Any
from app.data import guard
from app.data.engine import _jsonable, column_types, open_memory_sqlite
from app.data.guard import QueryLimits, SqlRejected
from app.data.names import name_key
from app.engine.direct_select import _kw, _lex, _Lex
from app.engine.expressions import same_value
#: 合并查询结果的查询快照里 source 记这个。数据源名只能是小写英文、数字和下划线(api/datasources.py 的
#: _NAME_PATTERN),永远撞不上它;证据面板、裁判摘录据此认出合并结果,不去按数据源名找遮罩
MERGE_SOURCE = "合并查询"
#: 一个合并节点最多几个输入。再多通常说明该在源库里先合并
MAX_INPUTS = 8
#: 别名(也就是内存库里的表名)最长几个字符
ALIAS_MAX = 32
_ALIAS = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
#: SQLite 的全部关键字(sqlite.org/lang_keywords.html),加上 true / false。用作表名要加引号才读得对,
#: 模型写合并 SQL 时多半不加——不如在起名时就挡住
_SQLITE_KEYWORDS = frozenset("""
abort action add after all alter always analyze and as asc attach autoincrement before begin between by cascade case
cast check collate column commit conflict constraint create cross current current_date current_time current_timestamp
database default deferrable deferred delete desc detach distinct do drop each else end escape except exclude exclusive
exists explain fail filter first following for foreign from full generated glob group groups having if ignore immediate
in index indexed initially inner insert instead intersect into is isnull join key last left like limit match
materialized natural no not nothing notnull null nulls of offset on or order others outer over partition plan pragma
preceding primary query raise range recursive references regexp reindex release rename replace restrict returning
right rollback row rows savepoint select set table temp temporary then ties to transaction trigger unbounded union
unique update using vacuum values view virtual when where window with without true false
""".split())
#: 快照里记的列类型 → 内存表的列亲和性。number 用 NUMERIC:数据源把 DECIMAL 落成的文本("3200.50")进来就是数,
#: 能直接参与运算;文本、日期一律 TEXT,'007' 的前导零不丢。拿不准的列不写类型,值原样存
_AFFINITY = {"number": "NUMERIC", "boolean": "INTEGER", "text": "TEXT", "date": "TEXT", "datetime": "TEXT",
"time": "TEXT"}
_KIND_LABEL = {"number": "数值", "boolean": "布尔值", "text": "文本", "date": "日期", "datetime": "日期时间",
"time": "时间"}
#: SQLite 表的 rowid 有三个名字;输入里恰好有同名的列时换下一个,三个都占了就不追逐格来历
_ROWID_NAMES = ("rowid", "_rowid_", "oid")
#: 补进选取列表的来历列。带引号写,和输入的列名撞不上(输入的列名里不会有这个前缀,撞了也只是对不上、不记来历)
_LINEAGE_COLUMN = "__agentlab_merge_row_{}"
SELECT_ONLY = "合并 SQL 只能是 SELECT 或 WITH 查询"
_TEMPLATE = re.compile(r"\{\{.*?\}\}", re.S)
#: 授权回调放行的动作:读表、调函数、递归公用表表达式。别的一律拒(ATTACH、PRAGMA、写)
_ALLOWED_ACTIONS = frozenset({sqlite3.SQLITE_SELECT, sqlite3.SQLITE_READ, sqlite3.SQLITE_FUNCTION,
sqlite3.SQLITE_RECURSIVE})
class MergeError(ValueError):
"""合并做不成。消息直接给人看:哪个输入、为什么、怎么改。"""
@dataclass(frozen=True)
class MergeInput:
"""一个输入:别名(内存库里的表名)、上游节点、它的查询快照。"""
alias: str
node_id: str
#: 上游查询快照的工件 id(tool 节点的 query_snapshot,或另一个合并节点的结果快照)
artifact: str
snapshot: dict[str, Any]
#: 上游节点在画布上的名字,只用在报错里
label: str = ""
@dataclass(frozen=True)
class Trace:
"""合并结果的一格来自哪个输入的哪一格(只走一跳;输入本身是合并结果时由调用方再追)。"""
alias: str
node_id: str | None
artifact: str | None
row: int
column: str
@dataclass
class MergeOutcome:
columns: list[str]
rows: list[list[Any]]
truncated: bool
#: 守卫规范过的那一条 SQL,执行的就是它
sql: str
column_types: dict[str, str]
warnings: list[dict[str, Any]]
#: 逐格来历(见模块说明);认不出写法、核对不上时为 None
lineage: dict[str, Any] | None
inputs: list[dict[str, Any]]
#: 底层数据源(输入本身是合并结果的,展开成它的数据源),去重保序
sources: list[str]
mask_columns: list[str] = field(default_factory=list)
def snapshot(self) -> dict[str, Any]:
"""存进工件库的查询快照:和数据源查询同形(columns、rows、row_count、truncated、sql、column_types、source),
另记 inputs、sources、warnings、lineage。不含耗时之类每次不同的东西:同样的输入、同样的 SQL 得到同一个工件,
节点重放、继续运行时快照 id 不变。"""
out: dict[str, Any] = {
"columns": self.columns, "rows": self.rows, "row_count": len(self.rows), "truncated": self.truncated,
"sql": self.sql, "source": MERGE_SOURCE, "inputs": self.inputs, "sources": self.sources,
"warnings": self.warnings,
}
if self.column_types:
out["column_types"] = self.column_types
if self.lineage is not None:
out["lineage"] = self.lineage
if self.mask_columns:
out["mask_columns"] = self.mask_columns
return out
# --------------------------------------------------------------------------
# 静态检查:别名、SQL(画图时校验用,执行时同一套规则)
# --------------------------------------------------------------------------
def alias_problem(alias: Any) -> str | None:
"""别名能不能当内存库里的表名。能用返回 None。"""
if not isinstance(alias, str) or not alias:
return "别名不能为空"
if len(alias) > ALIAS_MAX:
return f"别名「{alias[:ALIAS_MAX]}…」太长,最多 {ALIAS_MAX} 个字符"
if not _ALIAS.match(alias):
return f"别名「{alias}」不是合法的表名:只能用英文字母、数字和下划线,且不能以数字开头"
if alias.lower() in _SQLITE_KEYWORDS:
return f"别名「{alias}」是 SQL 关键字,用作表名必须加引号,容易写错。请换一个,例如 sales、visits"
if alias.lower().startswith("sqlite_"):
return f"别名「{alias}」以 sqlite_ 开头,这是 SQLite 保留给内部表的名字,请换一个"
return None
def sql_problem(sql: Any) -> str | None:
"""合并 SQL 有什么问题,没问题返回 None。{{ }} 运行时才有值,先按一个空值代进去检查写法。"""
if not isinstance(sql, str) or not sql.strip():
return "还没有填写合并 SQL"
try:
_checked(_TEMPLATE.sub("NULL", sql))
except MergeError as e:
return str(e)
return None
def _checked(sql: str) -> str:
"""守卫(SQLite 读法)规范过的那一条语句。只认 SELECT / WITH;守卫的原话写给数据源的,换成合并查询自己的说法。"""
if not sql or not sql.strip():
raise MergeError("还没有填写合并 SQL")
statements = guard.split_statements(sql, dialect="sqlite")
if len(statements) > 1:
raise MergeError(f"合并 SQL 只能写一条语句,这里有 {len(statements)} 条。请把它们合成一条 SELECT(可以用 WITH)")
try:
stmt = guard.check(sql, readonly=True, dialect="sqlite")
except SqlRejected as e:
# 只读拒绝的原话是「请在数据页另建一个可写的数据源」,合并查询用不上;结构变更(ATTACH、DROP)同理
if guard.is_write(sql, dialect="sqlite"):
raise MergeError(f"{SELECT_ONLY},不能写入、建表或连接别的库") from None
raise MergeError(f"合并 SQL 无法执行:{e}") from None
verb = guard.first_verb(stmt, dialect="sqlite")
if verb not in ("select", "with"):
raise MergeError(f"{SELECT_ONLY},当前是 {verb.upper() or '无法识别的语句'}")
return stmt
# --------------------------------------------------------------------------
# 执行
# --------------------------------------------------------------------------
def execute(inputs: list[MergeInput], sql: str, *, limits: QueryLimits | None = None) -> MergeOutcome:
"""把输入落进内存库,执行合并 SQL。做不成抛 MergeError(消息给人看)。"""
limits = limits or QueryLimits()
if not inputs:
raise MergeError("还没有配置输入:至少选择一个上游的查询节点")
if len(inputs) > MAX_INPUTS:
raise MergeError(f"输入最多 {MAX_INPUTS} 个,这里有 {len(inputs)} 个。请先在源库里合并一部分")
_check_aliases(inputs)
tables = {inp.alias: _input_columns(inp) for inp in inputs}
stmt = _checked(sql)
conn = open_memory_sqlite()
try:
rowids = {inp.alias: _load(conn, inp, tables[inp.alias]) for inp in inputs}
# 表落完就只读:授权回调之外的又一道(守卫、授权回调任何一处有漏洞,写也写不进去)
conn.execute("PRAGMA query_only = 1")
conn.set_authorizer(_authorize)
deadline = time.monotonic() + float(limits.timeout_seconds)
conn.set_progress_handler(lambda: 1 if time.monotonic() > deadline else 0, 1000)
columns, rows, truncated = _read(conn, stmt, limits)
plan = _lineage_plan(stmt, inputs, tables, rowids)
lineage = _lineage(conn, plan, columns, rows, inputs, tables) if plan is not None else None
except sqlite3.Error as e:
raise MergeError(_sqlite_reason(e, limits)) from None
finally:
conn.close()
warnings = _warnings(stmt, inputs, tables, rows, truncated, limits)
return MergeOutcome(
columns=columns, rows=rows, truncated=truncated, sql=stmt, column_types=column_types(columns, rows),
warnings=warnings, lineage=lineage, inputs=[_summary(inp, tables[inp.alias]) for inp in inputs],
sources=_sources(inputs), mask_columns=_masks(inputs, columns, plan),
)
def _check_aliases(inputs: list[MergeInput]) -> None:
seen: dict[str, str] = {}
for inp in inputs:
if problem := alias_problem(inp.alias):
raise MergeError(problem)
key = inp.alias.lower()
if key in seen:
raise MergeError(f"别名「{seen[key]}」和「{inp.alias}」只差大小写:SQLite 的表名不分大小写,会被当成同一张表。"
"请换一个别名")
seen[key] = inp.alias
def _who(inp: MergeInput) -> str:
return f"输入「{inp.alias}」" + (f"({inp.label})" if inp.label and inp.label != inp.alias else "")
def _input_columns(inp: MergeInput) -> list[str]:
snapshot = inp.snapshot
if not isinstance(snapshot, dict) or not isinstance(snapshot.get("columns"), list) \
or not isinstance(snapshot.get("rows"), list):
raise MergeError(f"{_who(inp)}不是查询结果:快照里没有列和行")
if snapshot.get("truncated"):
raise MergeError(
f"{_who(inp)}的查询结果被截断,只拿到了前 {len(snapshot['rows'])} 行。拿不完整的结果按键合并,缺的行不会报错,"
"只会让合并结果悄悄少算。请在源库里先聚合到合并需要的粒度(例如按日期和门店 GROUP BY),"
"或者加条件缩小范围,让结果不超过查询上限")
columns = [str(c) for c in snapshot["columns"]]
if not columns:
raise MergeError(f"{_who(inp)}的查询结果没有列")
seen: dict[str, str] = {}
for name in columns:
key = name_key(name)
if key in seen:
raise MergeError(f"{_who(inp)}的结果里有重名的列「{name}」:SQLite 的列名不分大小写,同名的列放不进一张表。"
"请在源查询中用 AS 给它们起不同的名字")
seen[key] = name
return columns
def _quote(name: str) -> str:
return '"' + name.replace('"', '""') + '"'
def _kind(snapshot: dict[str, Any], column: str) -> str | None:
"""这一列的类型:查询时记下的(column_types)优先;老快照没记的按值猜,猜不准不说。"""
types = snapshot.get("column_types")
if isinstance(types, dict) and isinstance(types.get(column), str):
return types[column]
columns = [str(c) for c in snapshot.get("columns") or []]
if column not in columns:
return None
at = columns.index(column)
kinds = {_value_kind(row[at]) for row in snapshot.get("rows") or []
if isinstance(row, list) and at < len(row) and row[at] is not None}
return next(iter(kinds)) if len(kinds) == 1 and None not in kinds else None
def _value_kind(value: Any) -> str | None:
if isinstance(value, bool):
return "boolean"
if isinstance(value, (int, float)):
return "number"
return "text" if isinstance(value, str) else None
def _cell_in(value: Any) -> Any:
"""快照里的值 → 写进内存库的值。快照是 JSON:布尔写成 0 / 1,对象和数组写成 JSON 文本。"""
if value is None or isinstance(value, (str, int, float)) and not isinstance(value, bool):
return value
if isinstance(value, bool):
return int(value)
if isinstance(value, (dict, list)):
return json.dumps(value, ensure_ascii=False, sort_keys=True)
return str(value)
def _load(conn: sqlite3.Connection, inp: MergeInput, columns: list[str]) -> str | None:
"""建表、按快照的顺序写入行,返回这张表的 rowid 用哪个名字(第 i 行的 rowid 是 i + 1)。"""
snapshot = inp.snapshot
decl = ", ".join(f"{_quote(c)} {_AFFINITY.get(_kind(snapshot, c) or '', '')}".rstrip() for c in columns)
conn.execute(f"CREATE TABLE {_quote(inp.alias)} ({decl})")
taken = {c.lower() for c in columns}
rowid = next((n for n in _ROWID_NAMES if n not in taken), None)
width = len(columns)
rows: list[list[Any]] = []
for i, row in enumerate(snapshot["rows"]):
if isinstance(row, dict):
row = [row.get(c) for c in columns]
if not isinstance(row, list) or len(row) != width:
raise MergeError(f"{_who(inp)}的快照第 {i} 行和列数对不上,快照可能已损坏")
rows.append([i + 1, *map(_cell_in, row)] if rowid else [_cell_in(v) for v in row])
names = ", ".join([*([rowid] if rowid else []), *map(_quote, columns)])
marks = ", ".join("?" * (width + (1 if rowid else 0)))
conn.executemany(f"INSERT INTO {_quote(inp.alias)} ({names}) VALUES ({marks})", rows)
return rowid
def _authorize(action: int, *_args: Any) -> int:
return sqlite3.SQLITE_OK if action in _ALLOWED_ACTIONS else sqlite3.SQLITE_DENY
def _read(conn: sqlite3.Connection, stmt: str, limits: QueryLimits) -> tuple[list[str], list[list[Any]], bool]:
"""逐行读、边读边判上限:和查询层(data.engine.run_query)同一个口径,到了行数或字节上限就停,记 truncated。"""
cursor = conn.execute(stmt)
columns = [str(d[0]) for d in cursor.description or []]
if not columns:
raise MergeError("合并 SQL 没有返回任何列")
rows: list[list[Any]] = []
size = 0
truncated = False
for row in cursor:
values = [_jsonable(v) for v in row]
rows.append(values)
size += len(json.dumps(values, ensure_ascii=False, default=str))
if len(rows) >= limits.max_rows or size >= limits.max_bytes:
truncated = True
break
return columns, rows, truncated
def _sqlite_reason(e: sqlite3.Error, limits: QueryLimits) -> str:
text = str(e)
if "interrupted" in text:
seconds = float(limits.timeout_seconds)
shown = f"{seconds:g}"
return f"合并超过 {shown} 秒被中断。请在源查询里先聚合,或在合并 SQL 中加条件缩小范围"
if "not authorized" in text or "readonly" in text:
return f"{SELECT_ONLY},只能读取输入表"
# SQLite 的原话(no such column: x、ambiguous column name: 日期)正是改 SQL 要的线索,原样带上
return f"合并 SQL 执行失败:{text}"
def _summary(inp: MergeInput, columns: list[str]) -> dict[str, Any]:
source = inp.snapshot.get("source")
return {"alias": inp.alias, "node_id": inp.node_id, "artifact": inp.artifact, "rows": len(inp.snapshot["rows"]),
"columns": columns, "source": str(source) if source else None}
def _sources(inputs: list[MergeInput]) -> list[str]:
out: list[str] = []
for inp in inputs:
nested = inp.snapshot.get("sources")
names = nested if inp.snapshot.get("source") == MERGE_SOURCE and isinstance(nested, list) \
else [inp.snapshot.get("source")]
out.extend(str(n) for n in names if n and str(n) not in out)
return out
# --------------------------------------------------------------------------
# 逐格来历
# --------------------------------------------------------------------------
#: 整条语句里出现就不追来历的词:子查询、集合运算、公用表表达式、分组、窗口、去重、NATURAL JOIN
_NO_LINEAGE = frozenset({"select", "union", "intersect", "except", "with", "recursive", "group", "having", "window",
"over", "distinct", "values", "natural", "filter"})
#: 聚合函数(名字后面紧跟「(」)。没有 GROUP BY 的聚合把整张表压成一行,选取项里的裸列取的是哪一行说不准
_AGGREGATES = frozenset({"count", "sum", "avg", "min", "max", "total", "group_concat", "string_agg", "median",
"json_group_array", "json_group_object", "jsonb_group_array", "jsonb_group_object"})
#: 不加引号的字面量关键字:选取项写 NULL、CURRENT_DATE 不是列
_LITERALS = frozenset({"null", "true", "false", "current_date", "current_time", "current_timestamp"})
#: 来源表后面紧跟这些词时,它们不是省略了 AS 的表别名
_AFTER_SOURCE = frozenset({"on", "using", "where", "order", "limit", "join", "left", "right", "full", "inner", "cross",
"outer", "indexed", "not", "natural", "group", "having", "window", "union", "intersect",
"except", "as"})
_JOIN_WORDS = frozenset({"join", "left", "right", "full", "inner", "cross", "natural"})
_TAIL = frozenset({"where", "order", "limit"})
@dataclass(frozen=True)
class _Source:
#: 输入别名(内存库里的表名)
alias: str
#: SQL 里怎么称呼它:写了表别名就是表别名,否则是表名本身
ref: str
@dataclass(frozen=True)
class _Plan:
sources: list[_Source]
#: 第 i 个结果列:(第几个来源, 输入里的列名);表达式、说不准的列是 None
columns: list[tuple[int, str] | None]
#: 在原 SQL 的哪个位置补来历列(顶层 FROM 前面)
insert_at: int
stmt: str
rowids: list[str]
def _punct(lex: _Lex | None, text: str) -> bool:
return lex is not None and lex.kind == "punct" and lex.text == text
def _named(lex: _Lex | None) -> bool:
return lex is not None and lex.kind in ("word", "name")
def _find(names: list[str], wanted: str) -> str | None:
key = name_key(wanted)
return next((n for n in names if name_key(n) == key), None)
def _lineage_plan(stmt: str, inputs: list[MergeInput], tables: dict[str, list[str]],
rowids: dict[str, str | None]) -> _Plan | None:
"""认出「单层 SELECT、选取项是输入表的列」这种写法;认不出返回 None(整份不记来历)。"""
toks = _lex(stmt)
if not toks or _kw(toks[0]) != "select":
return None
if any(_kw(t) in _NO_LINEAGE for t in toks[1:]):
return None
if any(_kw(t) in _AGGREGATES and _punct(nxt, "(") for t, nxt in zip(toks, toks[1:])):
return None
depth, from_at = 0, None
for i, t in enumerate(toks):
if _punct(t, "("):
depth += 1
elif _punct(t, ")"):
depth -= 1
elif depth == 0 and _kw(t) == "from":
from_at = i
break
if from_at is None:
return None
start = 2 if _kw(toks[1]) == "all" else 1
parsed = _from_clause(toks, from_at + 1, tables)
if parsed is None:
return None
sources, using = parsed
columns: list[tuple[int, str] | None] = []
for item in _split_items(toks[start:from_at]):
mapped = _item(item, sources, tables, using)
if mapped is None:
return None
columns.extend(mapped)
if any(rowids.get(s.alias) is None for s in sources):
return None
frm = toks[from_at]
return _Plan(sources=sources, columns=columns, insert_at=frm.end - len(frm.text), stmt=stmt,
rowids=[str(rowids[s.alias]) for s in sources])
def _split_items(toks: list[_Lex]) -> list[list[_Lex]]:
items: list[list[_Lex]] = [[]]
depth = 0
for t in toks:
if _punct(t, "("):
depth += 1
elif _punct(t, ")"):
depth -= 1
if depth == 0 and _punct(t, ","):
items.append([])
else:
items[-1].append(t)
return items
def _from_clause(toks: list[_Lex], i: int, tables: dict[str, list[str]]) -> tuple[list[_Source], set[str]] | None:
"""FROM 之后:来源表 [AS 别名],用逗号或 JOIN 连接,JOIN 后面可跟 ON 条件或 USING (列…);之后只许
WHERE / ORDER BY / LIMIT。来源只能是输入表(不认子查询、表函数、带库名的表)。"""
sources: list[_Source] = []
using: set[str] = set()
n = len(toks)
while True:
t = toks[i] if i < n else None
if not _named(t) or (t.kind == "word" and name_key(t.text) in _SQLITE_KEYWORDS):
return None
table = _find(list(tables), t.text)
if table is None or _punct(toks[i + 1] if i + 1 < n else None, ".") \
or _punct(toks[i + 1] if i + 1 < n else None, "("):
return None
ref, i = t.text, i + 1
nxt = toks[i] if i < n else None
if _kw(nxt) == "as":
if not _named(toks[i + 1] if i + 1 < n else None):
return None
ref, i = toks[i + 1].text, i + 2
elif _named(nxt) and (nxt.kind == "name" or name_key(nxt.text) not in _AFTER_SOURCE):
ref, i = nxt.text, i + 1
sources.append(_Source(alias=table, ref=ref))
word = _kw(toks[i]) if i < n else None
if word == "on":
depth, i = 0, i + 1
while i < n:
t = toks[i]
if _punct(t, "("):
depth += 1
elif _punct(t, ")"):
depth -= 1
elif depth == 0 and (_punct(t, ",") or _kw(t) in _JOIN_WORDS or _kw(t) in _TAIL):
break
i += 1
elif word == "using":
if not _punct(toks[i + 1] if i + 1 < n else None, "("):
return None
i += 2
while i < n and not _punct(toks[i], ")"):
if _named(toks[i]):
using.add(name_key(toks[i].text))
elif not _punct(toks[i], ","):
return None
i += 1
i += 1
if i >= n:
return sources, using
t = toks[i]
if _punct(t, ","):
i += 1
continue
word = _kw(t)
if word in _TAIL:
return sources, using
if word not in ("join", "left", "right", "full", "inner", "cross"):
return None
while i < n and _kw(toks[i]) in ("left", "right", "full", "inner", "cross", "outer"):
i += 1
if i >= n or _kw(toks[i]) != "join":
return None
i += 1
def _source_of(ref: str, sources: list[_Source]) -> int | None:
key = name_key(ref)
hits = [k for k, s in enumerate(sources) if name_key(s.ref) == key]
return hits[0] if len(hits) == 1 else None
def _item(item: list[_Lex], sources: list[_Source], tables: dict[str, list[str]],
using: set[str]) -> list[tuple[int, str] | None] | None:
"""一个选取项展开成几个结果列的来历。返回 None 表示整份不追(* 遇上 USING:SQLite 只出一份合并掉的列)。"""
if len(item) == 1 and _punct(item[0], "*"):
if using:
return None
return [(k, c) for k, s in enumerate(sources) for c in tables[s.alias]]
if len(item) == 3 and _named(item[0]) and _punct(item[1], ".") and _punct(item[2], "*"):
k = _source_of(item[0].text, sources)
return None if k is None else [(k, c) for c in tables[sources[k].alias]]
body = item
if len(item) >= 3 and _kw(item[-2]) == "as" and _named(item[-1]):
body = item[:-2]
elif len(item) >= 2 and _named(item[-1]) and _named(item[-2]) \
and not (item[-1].kind == "word" and name_key(item[-1].text) in _SQLITE_KEYWORDS):
body = item[:-1]
if len(body) == 1 and _named(body[0]):
word = body[0]
if word.kind == "word" and name_key(word.text) in _LITERALS:
return [None]
owners = [(k, col) for k, s in enumerate(sources) if (col := _find(tables[s.alias], word.text)) is not None]
if len(owners) != 1 or name_key(word.text) in using:
return [None]
return [owners[0]]
if len(body) == 3 and _named(body[0]) and _punct(body[1], ".") and _named(body[2]):
k = _source_of(body[0].text, sources)
col = _find(tables[sources[k].alias], body[2].text) if k is not None else None
return [(k, col) if k is not None and col is not None else None]
return [None]
def _hashable(row: list[Any]) -> str:
return json.dumps(row, ensure_ascii=False, sort_keys=True, default=str)
def _lineage(conn: sqlite3.Connection, plan: _Plan, columns: list[str], rows: list[list[Any]],
inputs: list[MergeInput], tables: dict[str, list[str]]) -> dict[str, Any] | None:
"""补上 rowid 再执行一次,逐行逐值核对,得出每一格来自哪个输入的哪一行。核对不上返回 None。"""
width = len(columns)
if len(plan.columns) != width or not any(plan.columns):
return None
extra = ", ".join(f"{_quote(s.ref)}.{rowid} AS {_quote(_LINEAGE_COLUMN.format(k))}"
for k, (s, rowid) in enumerate(zip(plan.sources, plan.rowids)))
head = plan.stmt[:plan.insert_at].rstrip()
try:
augmented = _checked(f"{head}, {extra} {plan.stmt[plan.insert_at:]}")
cursor = conn.execute(augmented)
except (MergeError, sqlite3.Error):
return None
got: list[tuple[Any, ...]] = []
for row in cursor:
if len(got) >= len(rows):
break
got.append(row)
if len(got) != len(rows) or any(len(g) != width + len(plan.sources) for g in got):
return None
if any([_jsonable(v) for v in g[:width]] != r for g, r in zip(got, rows)):
return None
origin: list[list[int | None]] = [[v - 1 if isinstance(v, int) and v >= 1 else None for v in g[width:]]
for g in got]
# 一模一样的两行结果:说不清哪一行来自哪一行输入
counts = Counter(_hashable(r) for r in rows)
for i, r in enumerate(rows):
if counts[_hashable(r)] > 1:
origin[i] = [None] * len(plan.sources)
snaps = {inp.alias: inp.snapshot for inp in inputs}
mapped: list[dict[str, Any] | None] = []
for j, target in enumerate(plan.columns):
if target is None:
mapped.append(None)
continue
k, column = target
snap = snaps[plan.sources[k].alias]
at = tables[plan.sources[k].alias].index(column)
ok = True
for i, row in enumerate(rows):
src = origin[i][k]
if src is None:
continue
if not (0 <= src < len(snap["rows"])) or not _same(row[j], _row(snap["rows"][src], tables[plan.sources[k].alias])[at]):
ok = False
break
mapped.append({"source": k, "column": column} if ok else None)
if not any(mapped):
return None
return {"sources": [{"input": s.alias, "ref": s.ref} for s in plan.sources], "columns": mapped, "rows": origin}
def _row(row: Any, columns: list[str]) -> list[Any]:
return [row.get(c) for c in columns] if isinstance(row, dict) else row
def _same(merged: Any, original: Any) -> bool:
if merged is None or original is None:
return merged is None and original is None
return same_value(merged, _cell_in(original))
def trace_cell(snapshot: Any, row: Any, column: Any) -> Trace | None:
"""合并结果的这一格来自哪个输入的哪一格;没有逐格来历返回 None。列按第一次出现的位置认(同 locate_cell)。"""
if not isinstance(snapshot, dict) or snapshot.get("source") != MERGE_SOURCE:
return None
lineage, columns = snapshot.get("lineage"), snapshot.get("columns")
if not isinstance(lineage, dict) or not isinstance(columns, list) or column not in columns:
return None
if isinstance(row, bool) or not isinstance(row, int):
return None
mapped, origin, sources = lineage.get("columns"), lineage.get("rows"), lineage.get("sources")
at = columns.index(column)
if not (isinstance(mapped, list) and isinstance(origin, list) and isinstance(sources, list)
and at < len(mapped) and 0 <= row < len(origin)):
return None
target = mapped[at]
if not isinstance(target, dict) or not isinstance(target.get("source"), int):
return None
k = target["source"]
line = origin[row]
if not isinstance(line, list) or not 0 <= k < len(line) or not 0 <= k < len(sources):
return None
src = line[k]
alias = (sources[k] or {}).get("input") if isinstance(sources[k], dict) else None
if isinstance(src, bool) or not isinstance(src, int) or not isinstance(alias, str):
return None
entry = next((e for e in snapshot.get("inputs") or [] if isinstance(e, dict) and e.get("alias") == alias), None)
if entry is None:
return None
return Trace(alias=alias, node_id=entry.get("node_id"), artifact=entry.get("artifact"), row=src,
column=str(target.get("column")))
def _masks(inputs: list[MergeInput], columns: list[str], plan: _Plan | None) -> list[str]:
"""合并结果要遮罩的列:各输入当时记下的遮罩列,加上来自遮罩列、换了名字的结果列(宁可多遮)。"""
hidden: dict[str, set[str]] = {}
names: list[str] = []
for inp in inputs:
recorded = inp.snapshot.get("mask_columns")
cols = {str(c).lower() for c in recorded} if isinstance(recorded, list) else set()
hidden[inp.alias] = cols
names.extend(str(c) for c in recorded or [] if isinstance(c, str))
if plan is not None and len(plan.columns) == len(columns):
for name, target in zip(columns, plan.columns):
if target is not None and target[1].lower() in hidden.get(plan.sources[target[0]].alias, set()):
names.append(name)
return sorted(dict.fromkeys(names))
# --------------------------------------------------------------------------
# 警告:合并键类型、行数放大、结果截断
# --------------------------------------------------------------------------
def _key_pairs(stmt: str, tables: dict[str, list[str]]) -> list[tuple[str, str, str, str]]:
"""合并条件里的键:ON / WHERE 里「列 = 列」两边落在不同的输入表上,以及 USING (列)。只用来提示,认不全不要紧。"""
toks = _lex(stmt, strict=False) or []
n = len(toks)
refs: dict[str, str] = {}
pairs: list[tuple[str, str, str, str]] = []
placed: list[str] = [] # 当前 FROM 列表里已经出现过的输入表(USING 找左边那张)
i = 0
while i < n:
word = _kw(toks[i])
if word in ("from", "join") or (_punct(toks[i], ",") and placed and i > 0 and _named(toks[i - 1])):
if word == "from":
placed = []
j = i + 1
t = toks[j] if j < n else None
table = _find(list(tables), t.text) if _named(t) else None
if table is not None and not _punct(toks[j + 1] if j + 1 < n else None, "."):
ref, j = t.text, j + 1
nxt = toks[j] if j < n else None
if _kw(nxt) == "as" and _named(toks[j + 1] if j + 1 < n else None):
ref, j = toks[j + 1].text, j + 2
elif _named(nxt) and (nxt.kind == "name" or name_key(nxt.text) not in _AFTER_SOURCE):
ref, j = nxt.text, j + 1
refs[name_key(ref)] = table
refs.setdefault(name_key(table), table)
if _kw(toks[j] if j < n else None) == "using" and _punct(toks[j + 1] if j + 1 < n else None, "("):
k = j + 2
while k < n and not _punct(toks[k], ")"):
if _named(toks[k]):
col = _find(tables[table], toks[k].text)
left = next((p for p in reversed(placed) if _find(tables[p], toks[k].text)), None)
if col is not None and left is not None:
pairs.append((left, str(_find(tables[left], toks[k].text)), table, col))
k += 1
placed.append(table)
i = j
continue
i += 1
def resolve(at: int, step: int) -> tuple[tuple[str, str] | None, int]:
"""从 at 开始(step=1 往右、-1 往左)读一个列引用:(表, 列)、它占了几个记号。"""
if step > 0:
seq = toks[at:at + 3]
else:
seq = toks[max(0, at - 2):at + 1]
if len(seq) == 3 and _named(seq[0]) and _punct(seq[1], ".") and _named(seq[2]):
table = refs.get(name_key(seq[0].text))
col = _find(tables[table], seq[2].text) if table else None
return ((table, col) if table and col else None), 3
one = toks[at] if 0 <= at < n else None
if _named(one):
owners = [(t, c) for t in tables if (c := _find(tables[t], one.text)) is not None]
return (owners[0] if len(owners) == 1 else None), 1
return None, 1
for i, t in enumerate(toks):
if not _punct(t, "="):
continue
prev = toks[i - 1] if i > 0 else None
# 「==」的第二个等号、<=、>=、!= 都不是比较的开头(分词把每个标点切成一个记号)
if prev is None or (prev.kind == "punct" and prev.text in "=<>!"):
continue
right_at = i + 2 if _punct(toks[i + 1] if i + 1 < n else None, "=") else i + 1
left, width = resolve(i - 1, -1)
before = toks[i - 1 - width] if i - 1 - width >= 0 else None
if before is not None and _punct(before, "."):
left = None # a.b.c 这种带库名的写法:不是输入表的列
right, rwidth = resolve(right_at, 1)
after = toks[right_at + rwidth] if right_at + rwidth < n else None
if after is not None and (_punct(after, ".") or _punct(after, "(")):
right = None
if left and right and left[0] != right[0]:
pairs.append((left[0], left[1], right[0], right[1]))
seen: set[frozenset[tuple[str, str]]] = set()
out: list[tuple[str, str, str, str]] = []
for a, ac, b, bc in pairs:
key = frozenset({(a, ac), (b, bc)})
if key not in seen:
seen.add(key)
out.append((a, ac, b, bc))
return out
def _example(snapshot: dict[str, Any], column: str) -> str | None:
columns = [str(c) for c in snapshot.get("columns") or []]
if column not in columns:
return None
at = columns.index(column)
for row in snapshot.get("rows") or []:
value = row[at] if isinstance(row, list) and at < len(row) else None
if value is not None:
return f"'{value}'" if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
return None
def _clash(a: str | None, b: str | None) -> bool:
"""两边的类型放在一起比会不会出错:数值对文本类,日期对日期时间('2026-05-01' 永远不等于 '2026-05-01T00:00:00')。"""
if not a or not b or a == b:
return False
numeric = {"number", "boolean"}
if (a in numeric) != (b in numeric):
return True
return {a, b} == {"date", "datetime"}
def _warnings(stmt: str, inputs: list[MergeInput], tables: dict[str, list[str]], rows: list[list[Any]],
truncated: bool, limits: QueryLimits) -> list[dict[str, Any]]:
snaps = {inp.alias: inp.snapshot for inp in inputs}
pairs = _key_pairs(stmt, tables)
out: list[dict[str, Any]] = []
for a, ac, b, bc in pairs:
ka, kb = _kind(snaps[a], ac), _kind(snaps[b], bc)
if not _clash(ka, kb):
continue
ea, eb = _example(snaps[a], ac), _example(snaps[b], bc)
left = f"{a}.{ac} 是{_KIND_LABEL.get(str(ka), ka)}" + (f"(例如 {ea})" if ea else "")
right = f"{b}.{bc} 是{_KIND_LABEL.get(str(kb), kb)}" + (f"(例如 {eb})" if eb else "")
if {ka, kb} == {"date", "datetime"}:
tail = ("日期和日期时间按文本比较永远不相等,这组键可能一行都匹配不上。请在源查询中统一格式,"
"或在合并 SQL 中用 date() 转换后再比较")
else:
tail = ("SQLite 比较时会做隐式转换:文本形式的编号(如 '001'、'01')都会等于数值 1,也可能完全匹配不上。"
"请在源查询中统一类型,或在合并 SQL 中用 CAST 明确转换")
out.append({"code": "key_type_mismatch", "keys": [f"{a}.{ac}", f"{b}.{bc}"],
"message": f"合并键类型不一致:{left},{right}。{tail}"})
largest = max(inputs, key=lambda inp: len(inp.snapshot["rows"]))
biggest = len(largest.snapshot["rows"])
if len(rows) > biggest:
keys: dict[str, list[str]] = {}
for a, ac, b, bc in pairs:
for alias, col in ((a, ac), (b, bc)):
if col not in keys.setdefault(alias, []):
keys[alias].append(col)
dups = [_duplicates(alias, snaps[alias], cols) for alias, cols in keys.items()]
detail = ";".join(d for d in dups if d)
out.append({"code": "rows_grew", "message": (
f"合并结果有 {len(rows)} 行,多于行数最多的输入「{largest.alias}」({biggest} 行):合并键可能不唯一,"
"同一行被重复匹配。" + (f"{detail}。" if detail else "")
+ "请检查合并条件是否覆盖了全部键(例如同时按日期和门店),或先在源库里聚合到相同粒度")})
if truncated:
out.append({"code": "result_truncated", "message": (
f"合并结果超过上限({limits.max_rows} 行或约 {max(1, limits.max_bytes // 1_000_000)} MB),只保留了前 "
f"{len(rows)} 行,下游拿到的不是完整结果。请在合并 SQL 中聚合或加条件缩小范围")})
return out
def _duplicates(alias: str, snapshot: dict[str, Any], cols: list[str]) -> str | None:
"""这个输入按合并键有没有重复:有的话举第一组,说一共几组。"""
columns = [str(c) for c in snapshot.get("columns") or []]
if not cols or any(c not in columns for c in cols):
return None
at = [columns.index(c) for c in cols]
counts = Counter(tuple(_hashable([row[j]]) for j in at) for row in snapshot.get("rows") or []
if isinstance(row, list))
repeated = [(key, n) for key, n in counts.items() if n > 1]
if not repeated:
return None
key, n = repeated[0]
shown = " / ".join(str(json.loads(v)[0]) for v in key)
more = f"等 {len(repeated)} 组" if len(repeated) > 1 else ""
return f"「{alias}」中按({'、'.join(cols)})有重复的键,例如 {shown} 出现 {n} 次{more}"
+132
View File
@@ -0,0 +1,132 @@
"""合并查询节点:读上游的查询快照,在库外按键合并,结果存成和数据源查询同形的查询快照。
**读的是工件库里那份完整结果。** 上游 tool 节点交回的 JSON 里带着查询快照的工件 id,这里按 id 取回快照(取回时复验
哈希),不读交给模型的那份文字:Agent 的消息历史里结果可能被截短,拿截短的文字合并会悄悄少行。上游是另一个合并
节点时同理,取它的结果快照。
**结果和数据源查询同形。** 快照有 columns、rows、truncated、sql、source(记为「合并查询」),台账里记一条 kind 为
query 的条目:报告照样按先后编成 Qn,[[v:Qn.rK.列]]、[[table:Qn …]] 和口径卡的 cell(nodes.<本节点>, 行, '列') 都按
原来的规则取值、核对。快照另记 inputs(别名、节点、各自的快照)和合并 SQL,证据面板据此展示合并的来源;逐格来历
记在 lineage 里(见 engine/merge_query.py),下钻据此指回输入的那一格。
**重放结果一致。** 快照里没有耗时这类每次不同的字段:节点重放、继续运行时同样的输入得到同一个工件 id、同一条
台账,报告里已经写下的出处不会因此变成另一份快照。
"""
from __future__ import annotations
import asyncio
import json
import time
from typing import Any
from app.core import artifact_store
from app.core.events import EventType
from app.data.guard import QueryLimits
from app.engine.context import NodeContext, NodeError
from app.engine.evidence import ledger_enabled, next_exec
from app.engine.merge_query import MERGE_SOURCE, MergeError, MergeInput, execute
from app.engine.state import GraphState
#: 合并警告进运行事件时的 code(log 事件)。运行详情、运行概况按它认出这是合并查询的警告
LOG_CODES = {"key_type_mismatch": "merge_key_type", "rows_grew": "merge_rows_grew",
"result_truncated": "merge_truncated"}
#: merge.end 事件里带几行预览:运行详情只是让人看一眼合并出了什么,完整结果在查询快照里
PREVIEW_ROWS = 5
#: 上游查询没查成时,tool 节点交回的原话的开头(engine/nodes/tools.py 的 QUERY_FAILED 同一组)
_FAILED_PREFIXES = ("查询失败", "SQL 被拒绝:")
def _input_payload(value: Any) -> dict[str, Any] | None:
"""上游节点的产出 → 带工件 id 的查询结果(tool 节点交回的是 JSON 文本,合并节点交回的是对象)。"""
data = value
if isinstance(data, str):
try:
data = json.loads(data)
except ValueError:
return None
return data if isinstance(data, dict) else None
async def _load_input(state: GraphState, ctx: NodeContext, alias: str, node_id: Any) -> MergeInput:
nodes = ctx.run.spec.node_map()
target = nodes.get(node_id) if isinstance(node_id, str) else None
if target is None:
raise NodeError(ctx.node.id, f"输入「{alias}」选择的节点「{node_id}」不存在,请在节点中重新选择")
who = f"输入「{alias}」({target.title})"
value = (state.get("nodes") or {}).get(target.id)
if value is None:
raise NodeError(ctx.node.id, f"{who}还没有运行结果:请确认它连在合并查询之前")
if isinstance(value, dict) and value.get("skipped"):
raise NodeError(ctx.node.id, f"{who}满足跳过条件被跳过了,没有查询结果可以合并")
if isinstance(value, dict) and value.get("failed"):
raise NodeError(ctx.node.id, f"{who}执行失败,没有查询结果可以合并:{value.get('error') or '原因未知'}")
if isinstance(value, str) and value.startswith(_FAILED_PREFIXES):
raise NodeError(ctx.node.id, f"{who}的查询没有成功,无法合并:{value[:300]}")
payload = _input_payload(value)
artifact = payload.get("artifact") if payload else None
if not isinstance(artifact, str) or not artifact:
raise NodeError(ctx.node.id, f"{who}的结果不是存成快照的查询结果。合并查询只能合并数据库查询(调用数据库查询工具的"
"「调用工具」节点)或另一个「合并查询」节点的结果")
try:
snapshot = await asyncio.to_thread(artifact_store.load, artifact)
except ValueError:
raise NodeError(ctx.node.id, f"{who}的查询快照与哈希不一致,疑似被修改,不能合并") from None
if snapshot is None:
raise NodeError(ctx.node.id, f"{who}的查询快照已不存在,无法合并。请重新运行它")
return MergeInput(alias=alias, node_id=target.id, artifact=artifact, snapshot=snapshot, label=target.title)
async def run_merge(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
inputs = ctx.config.get("inputs")
if not isinstance(inputs, dict) or not inputs:
raise NodeError(ctx.node.id, "「合并查询」还没有配置输入:至少选择一个上游的查询节点,并给它起一个别名")
# SQL 里允许 {{ }}(例如按运行输入筛日期),和 tool 节点的 SQL 一样先渲染;快照里存的是渲染之后、守卫规范过的那条
sql = ctx.render_str(str(ctx.config.get("sql") or ""), state)
loaded = [await _load_input(state, ctx, str(alias), node_id) for alias, node_id in inputs.items()]
started = time.perf_counter()
try:
# 内存 SQLite 是同步的:放进线程,大一点的合并不卡住事件循环(别的节点的事件照常流出去)
outcome = await asyncio.to_thread(execute, loaded, sql, limits=QueryLimits())
except MergeError as e:
raise NodeError(ctx.node.id, str(e)) from None
elapsed = int((time.perf_counter() - started) * 1000)
snapshot = outcome.snapshot()
try:
artifact = await artifact_store.put_json(snapshot, kind="query_snapshot", run_id=ctx.run.run_id,
node_id=ctx.node.id)
except Exception as e: # noqa: BLE001 - 存不下就没有出处可言,不能带着一份引用不了的结果往下走
raise NodeError(ctx.node.id, "合并结果没能存成查询快照(工件库写入失败),下游无法引用它。"
"请稍后点「继续运行」重试", retryable=True) from e
labels = {inp.alias: inp.label for inp in loaded}
inputs_view = [{**{k: v for k, v in item.items() if k != "columns"}, "label": labels.get(item["alias"]) or None}
for item in outcome.inputs]
ctx.emit(EventType.MERGE_END, inputs=inputs_view, sql=outcome.sql, rows=len(outcome.rows),
columns=outcome.columns, truncated=outcome.truncated, warnings=outcome.warnings,
preview_rows=outcome.rows[:PREVIEW_ROWS], duration_ms=elapsed, query_artifact=artifact,
lineage=outcome.lineage is not None)
for warning in outcome.warnings:
ctx.emit(EventType.LOG, level="warn", message=warning["message"],
code=LOG_CODES.get(warning["code"], "merge_warning"))
output: dict[str, Any] = {
"artifact": artifact, "columns": outcome.columns, "rows": outcome.rows, "row_count": len(outcome.rows),
"truncated": outcome.truncated, "sql": outcome.sql, "source": MERGE_SOURCE, "inputs": inputs_view,
"warnings": outcome.warnings,
}
if outcome.column_types:
output["column_types"] = outcome.column_types
updates: dict[str, Any] = {"nodes": {ctx.node.id: output}}
if ledger_enabled(ctx.run):
# 证据台账:和数据源查询同形的 query 条目,报告据此编 Qn;没有 via(外面不包工具快照)
updates["evidence"] = [{
"kind": "query", "node_id": ctx.node.id, "exec": next_exec(state, ctx.node.id), "artifact": artifact,
"source": MERGE_SOURCE, "columns": outcome.columns, "rows": len(outcome.rows),
"truncated": outcome.truncated,
}]
var_name = str(ctx.config.get("assign_to") or "").strip()
if var_name:
updates["vars"] = {var_name: output}
return updates
+1 -1
View File
@@ -330,7 +330,7 @@ _REF_HEAD = re.compile(r"^(vars|nodes|input)\.([A-Za-z_一-鿿][\w\-一-鿿]*)(.
_VIA = {"input": "input", "code": "code", "agent": "agent", "llm": "llm", "tool": "tool",
"transform": "transform", "metrics": "metric", "retrieve": "retrieval", "loop": "loop",
"human": "human", "validate": "validate", "supervisor": "team", "subgraph": "subgraph",
"memory": "memory"}
"memory": "memory", "merge": "merge"}
def _input_of(path: str, value: Any, state: GraphState, ctx: NodeContext,
+1
View File
@@ -23,6 +23,7 @@ class NodeType(StrEnum):
TOOL = "tool" # 直接调用一个工具
CODE = "code" # 沙箱里跑代码
MERGE = "merge" # 合并查询:几次查询的完整结果在库外按键合并(engine/merge_query.py)
BRANCH = "branch" # 条件分支
LOOP = "loop" # 循环 / 迭代
+4 -2
View File
@@ -53,7 +53,7 @@ _LLM_ONLY = ("prompt", "system", "skills", "use_history", "output_schema")
_MODEL_KEYS = ("provider", "model")
#: 报告的指令里照搬了这几类节点的文字,就要提醒:报告撰写自己会收集它们的证据
_EVIDENCE_TYPES = (NodeType.METRICS, NodeType.TOOL, NodeType.AGENT, NodeType.RETRIEVE, NodeType.CODE,
NodeType.SUBGRAPH)
NodeType.SUBGRAPH, NodeType.MERGE)
#: 写整段文字的节点:出具契约的 narrative 拼了其中几个的文字,就说不准该换哪一个
_WRITER_TYPES = (NodeType.LLM, NodeType.AGENT)
_QUESTION = re.compile(r"(?<![\w.])input\s*\.\s*question(?![\w-])")
@@ -93,7 +93,9 @@ class _Step:
def _queries(node: GraphNode) -> bool:
"""这个节点查库:报告撰写能把它查到的结果编成 Q1、Q2… 引用。只认数据源的查询工具,别的工具的
返回进不了目录。"""
返回进不了目录。合并查询的结果和数据源查询同形,同样编号。"""
if node.type == NodeType.MERGE:
return True
if node.type == NodeType.TOOL:
return str(node.config.get("tool") or "").startswith("db_query__")
if node.type == NodeType.AGENT:
+9
View File
@@ -340,6 +340,15 @@ def analyze(spec: GraphSpec) -> VariableReport:
node_id=node.id, node_label=label, field=field, expr=m.group(0),
))
# 合并查询的输入按节点 id 选,不写模板:不认这一条的话,变量表里看不出上游那条查询被谁用了。
# 选错的节点(不存在、不在上游、不是查询)由 validate_graph 报,这里只记引用
if node.type == "merge" and isinstance(cfg.get("inputs"), dict):
for alias, source in cfg["inputs"].items():
target = index.get(f"nodes.{source}") if isinstance(source, str) else None
if target is not None:
target.refs.append(VarRef(node_id=node.id, node_label=label, field=f"inputs.{alias}",
expr=f"nodes.{source}"))
for field, text in _iter_strings(cfg):
for m in _TEMPLATE_RE.finditer(text):
expr = m.group(1).strip()
+28
View File
@@ -0,0 +1,28 @@
-- 会员库(虚构):会员到店记录,和门店库是两个独立的库。门店编号同样是文本,但列名不同(store_code)。
-- 合并查询的端到端测试按「日期 + 门店」聚合到店人数,再和门店库的订单按键合并。数据只为测试编造。
CREATE TABLE visits (
visit_id INTEGER PRIMARY KEY,
member_id INTEGER NOT NULL,
store_code TEXT NOT NULL,
visit_date TEXT NOT NULL
);
INSERT INTO visits VALUES
(1, 101, 'S01', '2026-05-01'),
(2, 102, 'S01', '2026-05-01'),
(3, 103, 'S01', '2026-05-01'),
(4, 104, 'S01', '2026-05-01'),
(5, 105, 'S01', '2026-05-01'),
(6, 106, 'S01', '2026-05-01'),
(7, 201, 'S02', '2026-05-01'),
(8, 202, 'S02', '2026-05-01'),
(9, 203, 'S02', '2026-05-01'),
(10, 204, 'S02', '2026-05-01'),
(11, 101, 'S01', '2026-05-02'),
(12, 107, 'S01', '2026-05-02'),
(13, 108, 'S01', '2026-05-02'),
(14, 109, 'S01', '2026-05-02'),
(15, 110, 'S01', '2026-05-02'),
(16, 201, 'S02', '2026-05-02'),
(17, 205, 'S02', '2026-05-02'),
(18, 206, 'S02', '2026-05-02'),
(19, 207, 'S02', '2026-05-02');
+22
View File
@@ -0,0 +1,22 @@
-- 门店库(虚构):门店和销售订单。合并查询的端到端测试按「日期 + 门店」聚合订单数和销售额。
-- 门店编号是文本('S01'),金额是实数。数据只为测试编造,和任何真实门店无关。
CREATE TABLE stores (
store_id TEXT PRIMARY KEY,
name TEXT NOT NULL
);
CREATE TABLE orders (
order_id INTEGER PRIMARY KEY,
store_id TEXT NOT NULL REFERENCES stores (store_id),
order_date TEXT NOT NULL,
amount REAL NOT NULL
);
INSERT INTO stores VALUES ('S01', '湖滨店'), ('S02', '东站店');
INSERT INTO orders VALUES
(1, 'S01', '2026-05-01', 120.0),
(2, 'S01', '2026-05-01', 80.0),
(3, 'S01', '2026-05-01', 100.0),
(4, 'S02', '2026-05-01', 60.0),
(5, 'S02', '2026-05-01', 90.0),
(6, 'S01', '2026-05-02', 200.0),
(7, 'S01', '2026-05-02', 50.0),
(8, 'S02', '2026-05-02', 75.0);
+385
View File
@@ -0,0 +1,385 @@
"""合并查询的库外合并(engine/merge_query.py):几次查询的完整快照落进内存 SQLite,再按一条 SELECT / WITH 合并。
纯函数,不连库、不读工件:快照在测试里现造。端到端(真运行、证据面板、推断来源)见 test_merge_node.py。
场景是虚构的门店经营:门店库按「日期 + 门店」聚合的销售,会员库按同样粒度聚合的到店人数。
"""
from __future__ import annotations
import pytest
from app.data.guard import QueryLimits
from app.engine.merge_query import (
MERGE_SOURCE,
MergeError,
MergeInput,
alias_problem,
execute,
sql_problem,
trace_cell,
)
from app.engine.expressions import same_value
SALES_COLS = ["日期", "门店", "订单数", "销售额"]
SALES = [
["2026-05-01", "S01", 40, "3200.50"],
["2026-05-01", "S02", 25, "1875.00"],
["2026-05-02", "S01", 38, "3010.00"],
["2026-05-02", "S02", 30, "2400.00"],
]
VISITS_COLS = ["日期", "门店", "到店人数"]
VISITS = [
["2026-05-01", "S01", 160],
["2026-05-01", "S02", 90],
["2026-05-02", "S02", 100],
["2026-05-02", "S01", 150],
]
JOIN_SQL = ("SELECT s.日期, s.门店, s.订单数, s.销售额, v.到店人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 "
"ORDER BY s.日期, s.门店")
def snap(columns, rows, *, types=None, truncated=False, source="stores", **extra):
return {"columns": list(columns), "rows": [list(r) for r in rows], "row_count": len(rows), "truncated": truncated,
"sql": "SELECT …", "source": source, **({"column_types": types} if types is not None else {}), **extra}
def sales(**kw):
return MergeInput(alias="s", node_id="q_sales", artifact="a" * 64,
snapshot=snap(SALES_COLS, SALES, types={"日期": "text", "门店": "text", "订单数": "number",
"销售额": "number"}, **kw))
def visits(rows=VISITS, *, types=None, **kw):
return MergeInput(alias="v", node_id="q_visits", artifact="b" * 64,
snapshot=snap(VISITS_COLS, rows, types=types or {"日期": "text", "门店": "text", "到店人数": "number"},
source="members", **kw))
# --------------------------------------------------------------------------
# 合并本身
# --------------------------------------------------------------------------
def test_join_on_date_and_store():
out = execute([sales(), visits()], JOIN_SQL)
assert out.columns == ["日期", "门店", "订单数", "销售额", "到店人数"]
assert out.rows == [
["2026-05-01", "S01", 40, 3200.5, 160],
["2026-05-01", "S02", 25, 1875, 90],
["2026-05-02", "S01", 38, 3010, 150],
["2026-05-02", "S02", 30, 2400, 100],
]
# 数值列按数存:数据源把 DECIMAL 落成的文本("3200.50")合并之后能直接参与运算
assert out.column_types == {"日期": "text", "门店": "text", "订单数": "number", "销售额": "number",
"到店人数": "number"}
assert out.truncated is False and out.warnings == []
assert out.sql == JOIN_SQL
def test_with_is_allowed_and_text_keeps_leading_zeros():
a = MergeInput(alias="a", node_id="n1", artifact="c" * 64,
snapshot=snap(["会员号", "积分"], [["007", 10], ["070", 20]], types={"会员号": "text", "积分": "number"}))
out = execute([a], "WITH t AS (SELECT 会员号, 积分 FROM a) SELECT 会员号, 积分 * 2 AS 双倍 FROM t ORDER BY 会员号")
assert out.rows == [["007", 20], ["070", 40]]
def test_column_names_are_checked_not_treated_as_strings():
"""内存库同样关掉「双引号当字符串」:写错的列名要报错,不能静默得出一列常量。"""
with pytest.raises(MergeError, match="no such column"):
execute([sales()], 'SELECT "销售颔" FROM s')
@pytest.mark.parametrize("sql", [
"DELETE FROM s",
"UPDATE s SET 订单数 = 0",
"INSERT INTO s VALUES (1, 2, 3, 4)",
"DROP TABLE s",
"PRAGMA table_info(s)",
"ATTACH DATABASE '/tmp/x.db' AS x",
"WITH k AS (SELECT 1) DELETE FROM s",
"EXPLAIN SELECT * FROM s",
])
def test_only_select_and_with(sql):
assert sql_problem(sql) is not None
with pytest.raises(MergeError):
execute([sales()], sql)
def test_one_statement_only():
problem = sql_problem("SELECT * FROM s; SELECT * FROM v")
assert problem is not None and "一条" in problem
with pytest.raises(MergeError):
execute([sales(), visits()], "SELECT * FROM s; SELECT * FROM v")
def test_empty_sql():
assert "合并 SQL" in sql_problem(" ")
def test_static_check_tolerates_templates():
assert sql_problem("SELECT * FROM s WHERE 日期 >= {{ input.start | json }}") is None
@pytest.mark.parametrize("alias, ok", [
("s", True), ("sales_2026", True), ("_t", True),
("", False), ("1s", False), ("a-b", False), ("a b", False), ("订单", False),
("select", False), ("ORDER", False), ("join", False), ("sqlite_stat", False), ("x" * 40, False),
])
def test_alias_must_be_a_plain_table_name(alias, ok):
assert (alias_problem(alias) is None) is ok
def test_aliases_differing_only_in_case_collide():
a = MergeInput(alias="t", node_id="n1", artifact="c" * 64, snapshot=snap(["x"], [[1]]))
b = MergeInput(alias="T", node_id="n2", artifact="d" * 64, snapshot=snap(["x"], [[2]]))
with pytest.raises(MergeError, match="别名"):
execute([a, b], "SELECT * FROM t")
def test_duplicate_column_names_in_an_input_are_refused_with_a_fix():
a = MergeInput(alias="t", node_id="n1", artifact="c" * 64, snapshot=snap(["金额", "金额"], [[1, 2]]))
with pytest.raises(MergeError, match="重名"):
execute([a], "SELECT * FROM t")
# --------------------------------------------------------------------------
# 三道防护
# --------------------------------------------------------------------------
def test_a_truncated_input_is_refused():
"""被截断的输入只有前若干行:拿它按键合并,缺的行不会报错,只会悄悄少算。"""
with pytest.raises(MergeError) as e:
execute([sales(), visits(truncated=True)], JOIN_SQL)
message = str(e.value)
assert "「v」" in message and "截断" in message
assert "聚合" in message and "缩小范围" in message
def test_key_type_mismatch_is_warned():
"""门店编号一边是文本 '001',一边是数 1:SQLite 比较时会隐式转换,'001'、'01' 都等于 1。"""
stores = MergeInput(alias="s", node_id="q_sales", artifact="a" * 64,
snapshot=snap(["门店", "销售额"], [["001", 100], ["002", 200]],
types={"门店": "text", "销售额": "number"}))
traffic = MergeInput(alias="v", node_id="q_visits", artifact="b" * 64,
snapshot=snap(["门店", "到店人数"], [[1, 10], [2, 20]], types={"门店": "number", "到店人数": "number"},
source="members"))
out = execute([stores, traffic], "SELECT s.门店, s.销售额, v.到店人数 FROM s JOIN v ON s.门店 = v.门店")
[warning] = out.warnings
assert warning["code"] == "key_type_mismatch"
assert "s.门店" in warning["message"] and "v.门店" in warning["message"]
assert "'001'" in warning["message"] and "CAST" in warning["message"]
def test_key_type_mismatch_is_found_through_using():
stores = MergeInput(alias="s", node_id="n1", artifact="a" * 64,
snapshot=snap(["门店", "销售额"], [["1", 100]], types={"门店": "text", "销售额": "number"}))
traffic = MergeInput(alias="v", node_id="n2", artifact="b" * 64,
snapshot=snap(["门店", "到店人数"], [[1, 10]], types={"门店": "number", "到店人数": "number"}))
out = execute([stores, traffic], "SELECT 门店, 销售额, 到店人数 FROM s JOIN v USING (门店)")
assert [w["code"] for w in out.warnings] == ["key_type_mismatch"]
def test_matching_key_types_raise_no_warning():
out = execute([sales(), visits()], JOIN_SQL)
assert out.warnings == []
def test_more_rows_than_the_largest_input_is_warned():
"""两边都有一天同一门店出现两行(没聚合到同一粒度):多对多匹配,合并后行数被放大。"""
stores = sales()
stores.snapshot["rows"].append(["2026-05-01", "S01", 2, "100.00"])
doubled = [*VISITS, ["2026-05-01", "S01", 5]]
out = execute([stores, visits(doubled)], JOIN_SQL)
assert len(out.rows) == 7
grew = [w for w in out.warnings if w["code"] == "rows_grew"]
assert len(grew) == 1
message = grew[0]["message"]
assert "7 行" in message and "5 行" in message and "不唯一" in message
# 合并键认得出来时,说出是哪个输入、哪组键重复
assert "「v」" in message and "2026-05-01" in message and "S01" in message
def test_one_to_many_is_not_warned_as_growth():
"""一边的键唯一时,结果行数不会超过另一边:不报。"""
doubled = [*VISITS, ["2026-05-01", "S01", 5]]
out = execute([sales(), visits(doubled)], JOIN_SQL)
assert len(out.rows) == 5 and out.warnings == []
def test_result_over_the_row_limit_is_truncated_and_warned():
out = execute([sales(), visits()], "SELECT * FROM s CROSS JOIN v", limits=QueryLimits(max_rows=5))
assert out.truncated is True and len(out.rows) == 5
assert "result_truncated" in [w["code"] for w in out.warnings]
def test_byte_limit_matches_the_query_layer():
out = execute([sales(), visits()], "SELECT * FROM s CROSS JOIN v", limits=QueryLimits(max_bytes=100))
assert out.truncated is True and 0 < len(out.rows) < 16
def test_runaway_query_is_stopped():
endless = "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c) SELECT count(*) FROM c"
with pytest.raises(MergeError, match="秒"):
execute([sales()], endless, limits=QueryLimits(timeout_seconds=0.2))
# --------------------------------------------------------------------------
# 逐格来历:宁可不下钻,也不下钻错
# --------------------------------------------------------------------------
def _assert_traces_are_true(inputs, out):
"""凡是给出来历的格,都和它指向的那个输入的那一格是同一个值。"""
by_alias = {i.alias: i for i in inputs}
found = 0
for r, row in enumerate(out.rows):
for column in out.columns:
hit = trace_cell(out.snapshot(), r, column)
if hit is None:
continue
found += 1
source = by_alias[hit.alias].snapshot
value = source["rows"][hit.row][source["columns"].index(hit.column)]
assert same_value(row[out.columns.index(column)], value), (r, column, hit)
return found
def test_direct_columns_of_a_join_trace_to_the_input_rows():
inputs = [sales(), visits()]
out = execute(inputs, JOIN_SQL)
assert out.lineage is not None
# 2026-05-02 / S01 在会员库里是第 3 行(行号从 0 数),在门店库里是第 2 行
hit = trace_cell(out.snapshot(), 2, "到店人数")
assert (hit.alias, hit.row, hit.column) == ("v", 3, "到店人数")
hit = trace_cell(out.snapshot(), 2, "销售额")
assert (hit.alias, hit.row, hit.column) == ("s", 2, "销售额")
assert _assert_traces_are_true(inputs, out) == 20
def test_renamed_columns_still_trace():
inputs = [sales(), visits()]
out = execute(inputs, "SELECT s.门店 AS 店, v.到店人数 人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 "
"WHERE s.日期 = '2026-05-02' ORDER BY 1")
hit = trace_cell(out.snapshot(), 1, "人数")
assert (hit.alias, hit.row, hit.column) == ("v", 2, "到店人数")
assert _assert_traces_are_true(inputs, out) == 4
def test_computed_columns_have_no_trace():
inputs = [sales(), visits()]
out = execute(inputs, "SELECT s.门店, s.订单数 * 1.0 / v.到店人数 AS 转化率 FROM s JOIN v "
"ON s.日期 = v.日期 AND s.门店 = v.门店 ORDER BY s.日期, s.门店")
assert trace_cell(out.snapshot(), 0, "转化率") is None
assert trace_cell(out.snapshot(), 0, "门店").alias == "s"
@pytest.mark.parametrize("sql", [
"SELECT 日期, sum(订单数) AS 订单数 FROM s GROUP BY 日期",
"SELECT DISTINCT 门店 FROM s",
"SELECT 门店 FROM s UNION SELECT 门店 FROM v",
"SELECT 门店, (SELECT max(到店人数) FROM v) AS 峰值 FROM s",
"SELECT max(订单数) AS 订单数 FROM s",
"WITH t AS (SELECT * FROM s) SELECT 订单数 FROM t",
"SELECT 门店, 订单数, row_number() OVER (ORDER BY 订单数) AS 名次 FROM s",
"SELECT * FROM s NATURAL JOIN v",
])
def test_shapes_outside_the_grammar_have_no_trace_at_all(sql):
out = execute([sales(), visits()], sql)
assert out.lineage is None
assert all(trace_cell(out.snapshot(), r, c) is None for r in range(len(out.rows)) for c in out.columns)
def test_left_join_misses_have_no_trace_on_the_missing_side():
short = [["2026-05-01", "S01", 160]]
inputs = [sales(), visits(short)]
out = execute(inputs, "SELECT s.日期, s.门店, v.到店人数 FROM s LEFT JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 "
"ORDER BY s.日期, s.门店")
assert out.rows[1][2] is None
assert trace_cell(out.snapshot(), 1, "到店人数") is None
assert trace_cell(out.snapshot(), 1, "门店").alias == "s"
assert trace_cell(out.snapshot(), 0, "到店人数").row == 0
_assert_traces_are_true(inputs, out)
def test_star_expands_in_from_order():
inputs = [sales(), visits()]
out = execute(inputs, "SELECT * FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 ORDER BY s.日期, s.门店")
assert out.columns == [*SALES_COLS, *VISITS_COLS]
# 重名的列按第一次出现引用:「日期」指的是门店库那一列
assert trace_cell(out.snapshot(), 0, "日期").alias == "s"
assert trace_cell(out.snapshot(), 0, "到店人数").alias == "v"
_assert_traces_are_true(inputs, out)
def test_using_columns_written_bare_have_no_trace():
"""USING 合并掉的列不写表名时,SQLite 取的是哪一边说不准(RIGHT / FULL JOIN 时是两边合起来的值)。"""
out = execute([sales(), visits()], "SELECT 日期, 门店, 订单数, 到店人数 FROM s JOIN v USING (日期, 门店) ORDER BY 1, 2")
assert trace_cell(out.snapshot(), 0, "日期") is None
assert trace_cell(out.snapshot(), 0, "订单数").alias == "s"
def test_identical_output_rows_are_not_traced():
"""两行结果一模一样时,说不清哪一行来自哪一行输入:不给。"""
twins = [["2026-05-01", "S01", 7], ["2026-05-01", "S01", 7], ["2026-05-02", "S02", 9]]
inputs = [MergeInput(alias="v", node_id="n", artifact="b" * 64, snapshot=snap(VISITS_COLS, twins))]
out = execute(inputs, "SELECT 日期, 门店, 到店人数 FROM v")
assert trace_cell(out.snapshot(), 0, "到店人数") is None and trace_cell(out.snapshot(), 1, "到店人数") is None
assert trace_cell(out.snapshot(), 2, "到店人数").row == 2
def test_self_join_traces_each_side_separately():
inputs = [visits()]
out = execute(inputs, "SELECT a.门店, a.到店人数 AS 首日, b.到店人数 AS 次日 FROM v AS a JOIN v AS b "
"ON a.门店 = b.门店 AND a.日期 = '2026-05-01' AND b.日期 = '2026-05-02' ORDER BY a.门店")
assert out.rows == [["S01", 160, 150], ["S02", 90, 100]]
first, second = trace_cell(out.snapshot(), 0, "首日"), trace_cell(out.snapshot(), 0, "次日")
assert (first.alias, first.row) == ("v", 0) and (second.alias, second.row) == ("v", 3)
_assert_traces_are_true(inputs, out)
def test_a_rowid_column_in_the_input_does_not_confuse_the_trace():
rows = [[9, "S01", 1], [8, "S02", 2]]
inputs = [MergeInput(alias="t", node_id="n", artifact="b" * 64, snapshot=snap(["rowid", "门店", "值"], rows))]
out = execute(inputs, "SELECT 门店, 值 FROM t ORDER BY 门店 DESC")
assert out.rows == [["S02", 2], ["S01", 1]]
assert trace_cell(out.snapshot(), 0, "值").row == 1
_assert_traces_are_true(inputs, out)
# --------------------------------------------------------------------------
# 快照形状
# --------------------------------------------------------------------------
def test_snapshot_has_the_query_shape_plus_inputs():
out = execute([sales(), visits()], JOIN_SQL)
snapshot = out.snapshot()
for key in ("columns", "rows", "row_count", "truncated", "sql", "column_types"):
assert key in snapshot
assert snapshot["source"] == MERGE_SOURCE
assert snapshot["inputs"] == [
{"alias": "s", "node_id": "q_sales", "artifact": "a" * 64, "rows": 4, "columns": SALES_COLS, "source": "stores"},
{"alias": "v", "node_id": "q_visits", "artifact": "b" * 64, "rows": 4, "columns": VISITS_COLS,
"source": "members"},
]
assert snapshot["sources"] == ["stores", "members"]
# 内容确定:同样的输入、同样的 SQL,快照逐字相同(重放、继续运行得到同一个工件)
assert execute([sales(), visits()], JOIN_SQL).snapshot() == snapshot
def test_masked_input_columns_stay_masked_even_when_renamed():
members = MergeInput(alias="v", node_id="q_visits", artifact="b" * 64,
snapshot=snap(["门店", "手机号"], [["S01", "138****"]], source="members",
mask_columns=["手机号"]))
out = execute([sales(), members], "SELECT s.门店, v.手机号 AS 联系方式 FROM s JOIN v ON s.门店 = v.门店")
assert set(out.snapshot()["mask_columns"]) == {"手机号", "联系方式"}
def test_nested_merges_keep_the_underlying_sources():
first = execute([sales(), visits()], JOIN_SQL)
nested = MergeInput(alias="m", node_id="merge1", artifact="e" * 64, snapshot=first.snapshot())
out = execute([nested], "SELECT 日期, 门店, 到店人数 FROM m WHERE 门店 = 'S02'")
assert out.snapshot()["sources"] == ["stores", "members"]
hit = trace_cell(out.snapshot(), 0, "到店人数")
assert (hit.alias, hit.row) == ("m", 1)
+456
View File
@@ -0,0 +1,456 @@
"""合并查询节点(merge):画图时的校验、执行、证据链和下钻、助手说明。
端到端用两个独立的 SQLite 夹具(tests/fixtures/merge/):门店库的销售订单、会员库的到店记录。两条查询各自在自己的
库里聚合到「日期 + 门店」,合并查询在库外按键合并,口径卡用 cell(nodes.merge, …) 算转化率和客单价,报告引用合并
结果的单元格和口径卡的指标。断言报告里的数能追到两边的原始查询,下钻不指错格子。
不调用任何模型:报告的写作者换成脚本(monkeypatch MockChatModel._decide)。
"""
from __future__ import annotations
import asyncio
import json
import sqlite3
from pathlib import Path
import pytest
from httpx import ASGITransport, AsyncClient
from langchain_core.messages import AIMessage
from sqlalchemy import select
from app.core import artifact_store
from app.core.config import settings
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource, Run, RunEvent
from app.engine.evidence import iter_segments
from app.engine.merge_query import MERGE_SOURCE
from app.engine.runner import run_manager
from app.engine.schema import GraphSpec, NodeType, validate_graph
from app.main import app
FIXTURES = Path(__file__).parent / "fixtures" / "merge"
SALES_SQL = ("SELECT order_date AS 日期, store_id AS 门店, COUNT(*) AS 订单数, SUM(amount) AS 销售额 FROM orders "
"GROUP BY order_date, store_id ORDER BY order_date, store_id")
# 故意和门店库排成不同的顺序:合并结果的第 0 行在会员库的结果里是第 2 行
VISITS_SQL = ("SELECT visit_date AS 日期, store_code AS 门店, COUNT(*) AS 到店人数 FROM visits "
"GROUP BY visit_date, store_code ORDER BY visit_date DESC, store_code")
MERGE_SQL = ("SELECT s.日期, s.门店, s.订单数, s.销售额, v.到店人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 "
"ORDER BY s.日期, s.门店")
TEXT = ("5 月 1 日湖滨店到店 [[v:Q3.r0.到店人数]] 人,转化率 [[m:conversion]],客单价 [[m:aov]]。\n\n"
"[[table:Q3 cols=日期,门店,订单数,到店人数 rows=0-3]]")
# --------------------------------------------------------------------------
# 夹具
# --------------------------------------------------------------------------
@pytest.fixture
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
async def _source(name: str, path: Path, options: dict | None = None) -> str:
async with SessionLocal() as session:
row = (await session.execute(select(DataSource).where(DataSource.name == name))).scalar_one_or_none()
if row is None:
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True)
session.add(row)
row.database, row.kind, row.enabled, row.readonly, row.options = str(path), "sqlite", True, True, options or {}
await session.commit()
return row.id
@pytest.fixture
async def sources(tmp_path, engine_up):
"""把两个夹具脚本建成两个独立的 SQLite 库,各接成一个只读数据源。"""
ids = []
for name in ("stores", "members"):
path = tmp_path / f"{name}.db"
db = sqlite3.connect(path)
db.executescript((FIXTURES / f"{name}.sql").read_text("utf-8"))
db.commit()
db.close()
ids.append(await _source(f"merge_{name}", path))
for sid in ids:
await engines.invalidate(sid)
yield
for sid in ids:
await engines.invalidate(sid)
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def edge(a, b):
return {"source": a, "target": b}
def store_graph(*, sales_sql=SALES_SQL, visits_sql=VISITS_SQL, merge_sql=MERGE_SQL, visits_args=None, report=True,
inputs=None):
"""入口分两路查两个库,汇合到合并查询,再接口径卡、报告、成果。"""
nodes = [
node("start", "input"),
node("q_sales", "tool", tool="db_query__merge_stores", args={"sql": sales_sql}),
node("q_visits", "tool", tool="db_query__merge_members", args={"sql": visits_sql, **(visits_args or {})}),
node("merge", "merge", inputs=inputs or {"s": "q_sales", "v": "q_visits"}, sql=merge_sql),
node("card", "metrics", caliber="门店经营", caliber_version="v1", metrics=[
{"id": "conversion", "name": "转化率", "decimals": 4, "format": "percent_of_ratio",
"expression": "cell(nodes.merge, 0, '订单数') / cell(nodes.merge, 0, '到店人数')"},
{"id": "aov", "name": "客单价", "unit": "元", "decimals": 1,
"expression": "cell(nodes.merge, 0, '销售额') / cell(nodes.merge, 0, '订单数')"}]),
]
edges = [edge("start", "q_sales"), edge("start", "q_visits"), edge("q_sales", "merge"), edge("q_visits", "merge"),
edge("merge", "card")]
if report:
nodes += [node("write", "report", instructions="写门店日报", on_violation="flag"),
node("out", "output", fields=[{"name": "日报", "value": "{{ nodes.write.text }}"}])]
edges += [edge("card", "write"), edge("write", "out")]
else:
nodes.append(node("out", "output", fields=[{"name": "转化率", "value": "{{ nodes.card.metrics }}"}]))
edges.append(edge("card", "out"))
return {"nodes": nodes, "edges": edges}
def script(monkeypatch, text=TEXT):
from app.providers import mock_model
monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text))
async def wait(run_id: str) -> Run:
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run_id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
return row
raise AssertionError(f"run {run_id} 没有结束:{row.status}")
async def run_graph(monkeypatch, graph=None, text=TEXT) -> Run:
script(monkeypatch, text)
return await wait((await run_manager.start(graph=graph or store_graph(), input_payload={})).id)
async def events(run_id: str, etype: str, node_id: str | None = None) -> list[dict]:
async with SessionLocal() as session:
query = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype)
if node_id:
query = query.where(RunEvent.node_id == node_id)
return [e.data for e in (await session.execute(query.order_by(RunEvent.seq))).scalars()]
def load_doc(row: Run) -> dict:
doc_id = row.output["_evidence"]["doc_artifact"]
return json.loads((settings.data_dir / "artifacts" / doc_id[:2] / f"{doc_id}.json").read_text("utf-8"))
def seg_of(doc: dict, text: str, n: int = 0) -> dict:
return [s for s in iter_segments(doc) if s["text"] == text and s.get("cite")][n]
def alias_of(doc: dict, node_id: str) -> str:
[alias] = [a for a, e in doc["catalog"].items() if e.get("kind") == "query" and e.get("node_id") == node_id]
return alias
# --------------------------------------------------------------------------
# 画图时的校验
# --------------------------------------------------------------------------
def merge_spec(merge_cfg: dict, *, extra_nodes=(), extra_edges=()) -> GraphSpec:
base = store_graph(report=False)
nodes = [n if n["id"] != "merge" else node("merge", "merge", **merge_cfg) for n in base["nodes"]]
return GraphSpec.model_validate({"nodes": [*nodes, *extra_nodes], "edges": [*base["edges"], *extra_edges]})
def merge_errors(spec: GraphSpec) -> list:
return [i for i in validate_graph(spec).issues if i.node_id == "merge" and i.level == "error"]
def test_node_type_exists():
assert NodeType.MERGE == "merge"
def test_a_well_formed_merge_validates():
spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL})
assert merge_errors(spec) == []
def test_merge_of_a_merge_is_allowed():
spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL},
extra_nodes=[node("again", "merge", inputs={"m": "merge"}, sql="SELECT * FROM m")],
extra_edges=[edge("merge", "again")])
assert [i for i in validate_graph(spec).issues if i.node_id == "again" and i.level == "error"] == []
def test_inputs_are_required():
[issue] = merge_errors(merge_spec({"inputs": {}, "sql": MERGE_SQL}))
assert issue.field == "inputs" and issue.code == "merge.inputs_missing"
def test_input_pointing_to_a_missing_node():
[issue] = merge_errors(merge_spec({"inputs": {"s": "q_sales", "v": "nope"}, "sql": MERGE_SQL}))
assert issue.field == "inputs.v" and "不存在" in issue.message and issue.code == "merge.input_invalid"
@pytest.mark.parametrize("extra, why", [
(node("think", "llm", prompt="写点什么"), "不产出查询结果"),
(node("calc", "tool", tool="calculator", args={"expression": "1+1"}), "不是数据库查询工具"),
(node("asker", "agent", prompt="查一下", tools=["db_query__merge_stores"]), "Agent"),
])
def test_input_must_produce_a_query_result(extra, why):
spec = merge_spec({"inputs": {"s": "q_sales", "v": extra["id"]}, "sql": MERGE_SQL},
extra_nodes=[extra], extra_edges=[edge("start", extra["id"]), edge(extra["id"], "merge")])
[issue] = merge_errors(spec)
assert issue.field == "inputs.v" and why in issue.message and issue.code == "merge.input_invalid"
def test_agent_inputs_explain_how_to_fix():
extra = node("asker", "agent", prompt="查一下", tools=["db_query__merge_stores"])
spec = merge_spec({"inputs": {"s": "q_sales", "v": "asker"}, "sql": MERGE_SQL},
extra_nodes=[extra], extra_edges=[edge("start", "asker"), edge("asker", "merge")])
[issue] = merge_errors(spec)
assert "查询多次" in issue.message and "调用工具" in issue.message
def test_input_must_be_upstream():
lonely = node("q_other", "tool", tool="db_query__merge_stores", args={"sql": "SELECT 1 AS x"})
spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_other"}, "sql": MERGE_SQL},
extra_nodes=[lonely], extra_edges=[edge("start", "q_other")])
[issue] = merge_errors(spec)
assert issue.field == "inputs.v" and "上游" in issue.message
@pytest.mark.parametrize("alias", ["1s", "a-b", "select", "sqlite_x"])
def test_alias_must_be_a_legal_table_name(alias):
[issue] = merge_errors(merge_spec({"inputs": {"s": "q_sales", alias: "q_visits"}, "sql": MERGE_SQL}))
assert issue.field == f"inputs.{alias}" and issue.code == "merge.alias_invalid" and alias in issue.message
@pytest.mark.parametrize("sql, why", [
("", "还没有填写"),
("DELETE FROM s", "SELECT 或 WITH"),
("PRAGMA table_info(s)", "SELECT 或 WITH"),
("SELECT * FROM s; SELECT * FROM v", "一条"),
])
def test_sql_must_be_one_select_or_with(sql, why):
[issue] = merge_errors(merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": sql}))
assert issue.field == "sql" and why in issue.message and issue.code == "merge.sql_invalid"
def test_merge_inputs_count_as_references_to_the_upstream_nodes():
from app.engine.variables import analyze
report = analyze(merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL}))
refs = {v.path: v.refs for v in report.variables}
assert any(r.node_id == "merge" for r in refs["nodes.q_sales"])
def test_labels_and_governance_know_the_type():
from app.engine.governance import _traceable
from app.engine.labels import field_label, type_label
from app.engine.upgrade import _queries
spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL})
merge = spec.node_map()["merge"]
assert type_label("merge") == "合并查询"
assert field_label("sql", "merge") == "合并 SQL" and field_label("inputs", "merge") == "输入"
assert _traceable(merge) and _queries(merge)
# --------------------------------------------------------------------------
# 执行与证据链
# --------------------------------------------------------------------------
async def test_end_to_end_two_databases(monkeypatch, sources, client):
row = await run_graph(monkeypatch)
assert row.status == "succeeded", row.error
[finished] = await events(row.id, "node.finished", "merge")
output = artifact_store.load(finished["artifact"])
assert output["columns"] == ["日期", "门店", "订单数", "销售额", "到店人数"]
assert output["rows"] == [["2026-05-01", "S01", 3, 300, 6], ["2026-05-01", "S02", 2, 150, 4],
["2026-05-02", "S01", 2, 250, 5], ["2026-05-02", "S02", 1, 75, 4]]
assert output["source"] == MERGE_SOURCE and output["warnings"] == []
# 台账:一条和数据源查询同形的 query 条目,报告据此编号
[entry] = finished["evidence"]
assert entry["kind"] == "query" and entry["source"] == MERGE_SOURCE and entry["artifact"] == output["artifact"]
assert entry["rows"] == 4 and entry["truncated"] is False
snapshot = artifact_store.load(entry["artifact"])
assert [i["alias"] for i in snapshot["inputs"]] == ["s", "v"]
assert snapshot["sources"] == ["merge_stores", "merge_members"]
assert snapshot["sql"] == MERGE_SQL
# 运行事件:合并的输入、SQL、行数,运行详情据此展示
[done] = await events(row.id, "merge.end", "merge")
assert done["sql"] == MERGE_SQL and done["rows"] == 4 and done["warnings"] == []
assert [(i["alias"], i["node_id"], i["rows"], i["source"]) for i in done["inputs"]] == [
("s", "q_sales", 4, "merge_stores"), ("v", "q_visits", 4, "merge_members")]
assert done["query_artifact"] == entry["artifact"]
doc = load_doc(row)
q3 = doc["catalog"]["Q3"]
assert q3["node_id"] == "merge" and q3["source"] == MERGE_SOURCE and q3["artifact"] == entry["artifact"]
assert {alias_of(doc, "q_sales"), alias_of(doc, "q_visits")} == {"Q1", "Q2"}
metric = doc["catalog"]["m:conversion"]
assert metric["value"] == 0.5 and metric["rendered"] == "50.00%"
# 口径卡的两个输入都按 cell() 落在合并结果的那一格上,并按快照核对过
card = artifact_store.load(metric["artifact"])
conversion = next(m for m in card["metrics"] if m["id"] == "conversion")
assert [(i["via"], i["status"], i["artifact"], i["locator"]) for i in conversion["inputs"]] == [
("tool_cell", "verified", entry["artifact"], {"row": 0, "column": "订单数"}),
("tool_cell", "verified", entry["artifact"], {"row": 0, "column": "到店人数"})]
text = row.output["日报"]
assert "到店 6 人" in text and "50.00%" in text and "100.0元" in text
async def test_cell_segment_traces_to_both_original_queries(monkeypatch, sources, client):
row = await run_graph(monkeypatch)
assert row.status == "succeeded", row.error
doc = load_doc(row)
qv = alias_of(doc, "q_visits")
qs = alias_of(doc, "q_sales")
seg = seg_of(doc, "6")
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}")).json()
merge_step, visits_step = body["chain"]
assert merge_step["step"] == "query" and merge_step["alias"] == "Q3" and merge_step["source"] == MERGE_SOURCE
assert merge_step["hash_ok"] is True and merge_step["sealed"] is True
assert "mask_note" not in merge_step # 合并查询不是数据源,不能说它「已不存在」
merged = merge_step["merge"]
assert merged["sql"] == MERGE_SQL and merged["warnings"] == []
assert [(i["alias"], i["node_id"], i["query"], i["rows"], i["source"], i["sealed"]) for i in merged["inputs"]] == [
("s", "q_sales", qs, 4, "merge_stores", True), ("v", "q_visits", qv, 4, "merge_members", True)]
assert merged["traced"] == [{"cell": [0, "到店人数"], "input": "v", "query": qv, "row": 2, "column": "到店人数"}]
# 接着给出会员库那次查询,高亮的正是那一格,值相同
assert visits_step["step"] == "query" and visits_step["alias"] == qv and visits_step["merged_into"] == "Q3"
assert visits_step["highlight"]["cells"] == [[2, "到店人数"]]
at = visits_step["row_index"].index(2)
assert visits_step["rows"][at][visits_step["columns"].index("到店人数")] == 6
async def test_metric_segment_traces_through_the_merge(monkeypatch, sources, client):
row = await run_graph(monkeypatch)
doc = load_doc(row)
qs, qv = alias_of(doc, "q_sales"), alias_of(doc, "q_visits")
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_of(doc, '50.00%')['id']}")).json()
metric, *inputs = body["chain"][:3]
assert metric["step"] == "metric" and metric["value"] == 0.5
assert [(i["via"], i["status"], i["query"], i["cell"]) for i in inputs] == [
("tool_cell", "verified", "Q3", "Q3.r0.订单数"), ("tool_cell", "verified", "Q3", "Q3.r0.到店人数")]
steps = [s for s in body["chain"] if s["step"] == "query"]
assert [s["alias"] for s in steps] == ["Q3", qs, qv]
merge_step, sales_step, visits_step = steps
assert sorted(t["input"] for t in merge_step["merge"]["traced"]) == ["s", "v"]
assert sales_step["highlight"]["cells"] == [[0, "订单数"]] and sales_step["merged_into"] == "Q3"
assert visits_step["highlight"]["cells"] == [[2, "到店人数"]]
async def test_provenance_of_a_merged_cell_follows_the_trace(monkeypatch, sources, client):
"""手工源上的格:追到会员库那次查询,结论和直接引用那一格相同(not_upload),并记下经过的合并。"""
row = await run_graph(monkeypatch)
doc = load_doc(row)
seg = seg_of(doc, "6")
out = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}/provenance")).json()
assert (out["status"], out["reason"]["code"]) == ("none", "not_upload")
assert out["cell"] == {"alias": "Q3", "row": 0, "column": "到店人数", "artifact": doc["catalog"]["Q3"]["artifact"]}
assert out["merge"] == [{"alias": "Q3", "node_id": "merge", "input": "v", "query": alias_of(doc, "q_visits"),
"row": 2, "column": "到店人数"}]
async def test_a_computed_merge_column_has_no_cell_lineage(monkeypatch, sources, client):
sql = ("SELECT s.日期, s.门店, s.订单数 + v.到店人数 AS 合计 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 "
"ORDER BY s.日期, s.门店")
graph = store_graph(merge_sql=sql)
graph["nodes"] = [n for n in graph["nodes"] if n["id"] != "card"]
graph["edges"] = [e for e in graph["edges"] if "card" not in (e["source"], e["target"])] + [edge("merge", "write")]
row = await run_graph(monkeypatch, graph, text="合计 [[v:Q3.r0.合计]]。")
assert row.status == "succeeded", row.error
doc = load_doc(row)
seg = seg_of(doc, "9")
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}")).json()
[merge_step] = body["chain"]
[traced] = merge_step["merge"]["traced"]
assert traced["input"] is None and traced["query"] is None and "逐格来历" in traced["note"]
out = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}/provenance")).json()
assert (out["status"], out["reason"]["code"], out["version"]) == ("none", "merge_no_lineage", None)
assert out["merge"] == [{"alias": "Q3", "node_id": "merge", "input": None, "query": None, "row": None,
"column": None}]
async def test_a_truncated_input_fails_the_node(monkeypatch, sources):
row = await run_graph(monkeypatch, store_graph(visits_args={"limit": 2}, report=False))
assert row.status == "failed"
[failed] = await events(row.id, "node.failed", "merge")
assert "「v」" in failed["error"] and "截断" in failed["error"] and "聚合" in failed["error"]
async def test_warnings_reach_the_output_and_the_events(monkeypatch, sources):
"""门店编号一边是文本 '01'、一边是数 1;只按门店合并、没按日期:两道警告都要看得见。"""
sales_sql = ("SELECT order_date AS 日期, substr(store_id, 2) AS 门店, COUNT(*) AS 订单数, SUM(amount) AS 销售额 "
"FROM orders GROUP BY order_date, store_id ORDER BY 1, 2")
visits_sql = ("SELECT visit_date AS 日期, CAST(substr(store_code, 2) AS INTEGER) AS 门店, COUNT(*) AS 到店人数 "
"FROM visits GROUP BY visit_date, store_code ORDER BY 1, 2")
merge_sql = "SELECT s.门店, s.订单数, s.销售额, v.到店人数 FROM s JOIN v ON s.门店 = v.门店"
row = await run_graph(monkeypatch, store_graph(sales_sql=sales_sql, visits_sql=visits_sql, merge_sql=merge_sql,
report=False))
assert row.status == "succeeded", row.error
[finished] = await events(row.id, "node.finished", "merge")
output = artifact_store.load(finished["artifact"])
assert sorted(w["code"] for w in output["warnings"]) == ["key_type_mismatch", "rows_grew"]
[done] = await events(row.id, "merge.end", "merge")
assert sorted(w["code"] for w in done["warnings"]) == ["key_type_mismatch", "rows_grew"]
logs = [e for e in await events(row.id, "log", "merge") if e.get("level") == "warn"]
assert sorted(e["code"] for e in logs) == ["merge_key_type", "merge_rows_grew"]
assert all(e["message"] for e in logs)
async def test_rerunning_the_node_on_the_same_inputs_gives_the_same_snapshot(monkeypatch, sources):
"""重放、继续运行时节点从头再执行:同样的输入得到同一份快照、同一条台账,下游的出处不变。"""
from app.engine.context import NodeContext, RunContext
from app.engine.nodes.merge import run_merge
row = await run_graph(monkeypatch, store_graph(report=False))
assert row.status == "succeeded", row.error
outputs = {}
for nid in ("q_sales", "q_visits"):
[finished] = await events(row.id, "node.finished", nid)
outputs[nid] = artifact_store.load(finished["artifact"])
spec = GraphSpec.model_validate(store_graph(report=False))
ctx = NodeContext(node=spec.node_map()["merge"],
run=RunContext(run_id=row.id, thread_id=row.id, spec=spec, agent_limits={}))
state = {"nodes": outputs, "trail": [], "evidence": []}
first, second = await run_merge(state, ctx), await run_merge(state, ctx)
assert first == second
assert first["evidence"][0]["artifact"] == first["nodes"]["merge"]["artifact"]
# --------------------------------------------------------------------------
# 助手
# --------------------------------------------------------------------------
def test_node_reference_explains_when_to_use_merge():
from app.api.copilot import NODE_REFERENCE
assert "- merge:" in NODE_REFERENCE
block = NODE_REFERENCE.split("- merge:", 1)[1].split("\n- ", 1)[0]
for phrase in ("inputs", "sql", "不同的库", "同一个库", "一条 SQL", "口径卡", "截断", "cell(nodes."):
assert phrase in block, phrase