mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-19 09:42:04 +00:00
合并查询节点(一):新增 merge 节点和执行器——几次查询的完整快照落进内存 SQLite 按键合并,结果存成同形的查询快照,截断的输入拒绝、键类型不一致和行数放大给警告,并记下逐格来历
This commit is contained in:
@@ -48,6 +48,9 @@ class EventType(StrEnum):
|
||||
# 记忆读写。以前发的是 info 日志,而解码器丢弃所有 info——于是往长期记忆里
|
||||
# 写东西这件事在界面上是隐形的。让 Copilot 主动记之后,这条不可接受
|
||||
MEMORY_END = "memory.end"
|
||||
# 合并查询做完了:哪几个输入(别名、节点、行数、数据源)、哪条合并 SQL、得到几行、有哪些警告。
|
||||
# 运行详情据此展示这一步;完整结果在 query_artifact 指向的查询快照里
|
||||
MERGE_END = "merge.end"
|
||||
SANDBOX_START = "sandbox.start"
|
||||
SANDBOX_END = "sandbox.end"
|
||||
|
||||
|
||||
@@ -277,6 +277,15 @@ def open_checked_sqlite(path: str, *, readonly: bool = True, immutable: bool = F
|
||||
factory=_DqsOffConnection)
|
||||
|
||||
|
||||
def open_memory_sqlite() -> sqlite3.Connection:
|
||||
"""一个内存 SQLite 连接,DQS 已关(合并查询把几次查询的结果落进来再合并,engine/merge_query.py)。
|
||||
|
||||
合并 SQL 同样是模型写的:双引号里的列名写错一个字,默认连接会把它当成字符串常量,`SUM("销售颔")` 得 0,
|
||||
`ON s."门店" = v."门店 "` 一行都对不上却不报错。所以和连接层共用 _DqsOffConnection,关不掉就不交出连接。
|
||||
"""
|
||||
return sqlite3.connect(":memory:", factory=_DqsOffConnection)
|
||||
|
||||
|
||||
def _sqlite_extra() -> dict[str, Any]:
|
||||
"""每个 SQLite engine 都带的连接参数:关 DQS。"""
|
||||
return {"connect_args": {"factory": _DqsOffConnection}}
|
||||
|
||||
@@ -11,7 +11,7 @@ from langgraph.graph import END, START, StateGraph
|
||||
from app.core.errors import describe_exception, raw_detail
|
||||
from app.core.events import EventType
|
||||
from app.engine.context import NodeContext, NodeError, RunContext
|
||||
from app.engine.nodes import control, human, io, knowledge, llm, metrics, multi, report, tools
|
||||
from app.engine.nodes import control, human, io, knowledge, llm, merge, metrics, multi, report, tools
|
||||
from app.engine.schema import GraphNode, GraphSpec, NodeType, back_edges, innermost_loops
|
||||
from app.engine.state import GraphState
|
||||
|
||||
@@ -28,6 +28,7 @@ RUNNERS: dict[NodeType, NodeRunner] = {
|
||||
NodeType.SUBGRAPH: multi.run_subgraph,
|
||||
NodeType.TOOL: tools.run_tool,
|
||||
NodeType.CODE: tools.run_code,
|
||||
NodeType.MERGE: merge.run_merge,
|
||||
NodeType.BRANCH: control.run_branch,
|
||||
NodeType.LOOP: control.run_loop,
|
||||
NodeType.HUMAN: human.run_human,
|
||||
|
||||
@@ -237,7 +237,7 @@ REPORT_POLICY_ALSO: dict[str, tuple[str, ...]] = {"claims": (CLAIMS_JUDGE,)}
|
||||
JUDGE_UNLIMITED = "不设上限,费用只受句数、时长上限和每日上限约束"
|
||||
|
||||
#: 产出本身就是证据、或者由系统确定地算出来的节点。成果字段取它们的值,不算「别的节点写的字」
|
||||
_TRACEABLE = (NodeType.REPORT, NodeType.METRICS, NodeType.INPUT, NodeType.TOOL, NodeType.RETRIEVE)
|
||||
_TRACEABLE = (NodeType.REPORT, NodeType.METRICS, NodeType.INPUT, NodeType.TOOL, NodeType.RETRIEVE, NodeType.MERGE)
|
||||
_WRITERS = (NodeType.LLM, NodeType.AGENT, NodeType.SUPERVISOR)
|
||||
#: 往对话里写消息的节点(emit_message 没关时),「最后一条消息」就是它们里最后跑的那个写的
|
||||
_EMITTERS = (*_WRITERS, NodeType.REPORT)
|
||||
|
||||
@@ -18,7 +18,7 @@ from typing import Any
|
||||
#: 节点类型的叫法,和前端 lib/terms.ts 的 NODE_TYPE_LABEL 一致
|
||||
TYPE_LABEL: dict[str, str] = {
|
||||
"input": "输入", "output": "成果", "llm": "模型调用", "agent": "Agent",
|
||||
"supervisor": "多 Agent 协作", "tool": "调用工具", "code": "沙箱代码",
|
||||
"supervisor": "多 Agent 协作", "tool": "调用工具", "code": "沙箱代码", "merge": "合并查询",
|
||||
"branch": "条件分支", "loop": "循环", "subgraph": "子工作流",
|
||||
"memory": "长期记忆", "retrieve": "知识检索", "transform": "数据整形",
|
||||
"human": "人工审批", "validate": "结构校验", "metrics": "口径卡", "report": "报告撰写",
|
||||
@@ -79,6 +79,7 @@ _FIELD_BY_TYPE: dict[str, dict[str, str]] = {
|
||||
"human": {"mode": "审批方式", "title": "标题", "message": "展示给审批人的内容", "draft": "草稿内容",
|
||||
"stop_on_reject": "驳回即终止运行"},
|
||||
"validate": {"source": "待校验内容", "fail_fast": "校验失败即中断"},
|
||||
"merge": {"inputs": "输入", "sql": "合并 SQL"},
|
||||
}
|
||||
|
||||
#: 出具契约里的键(canvas/issues.ts 的 CONTRACT_KEYS、SUB_KEYS)
|
||||
|
||||
@@ -0,0 +1,879 @@
|
||||
"""合并查询(merge 节点):几次查询的完整结果落进内存 SQLite,再用一条 SELECT / WITH 按键合并。
|
||||
|
||||
**为什么要有它。** 跨查询组合原来只有三种:口径卡用 cell() 组合几个单值;tool 节点的 SQL 里写 {{ vars.x }},把前一条
|
||||
查询的结果代进后一条;同一个库里写一条 JOIN / WITH。按行合并两次查询,尤其数据在两个库里时,没有路可走:用沙箱代码
|
||||
合并,出处链断在代码节点上,喂给口径卡时只能提示「核对不了出处」;让模型把 id 从一条查询抄进下一条,会抄错,结果被
|
||||
截断后还会漏。
|
||||
|
||||
**为什么不做跨库联邦查询。** 联邦查询要把两边的明细拉到一处,数据量、方言、权限一样都控制不住。方案是先在各自的库里
|
||||
聚合到共同的键和粒度(日期 + 门店),再由这里在库外按键合并:两边进来的都是已经聚合过的小结果。
|
||||
|
||||
**为什么落成 SQLite、用标准库 sqlite3。** 照搬 data/tabular.py:SQLite 是现成的 SQL 引擎,守卫有它的读法(guard
|
||||
的 _SQLITE),证据下钻的识别器(direct_select)也按它分词,不必为这一步引入 DuckDB 再多一种方言。连接同样关掉 DQS
|
||||
(data.engine.open_memory_sqlite):双引号里的列名写错要报错,不能当成字符串常量。
|
||||
|
||||
**守卫和三道防护。**
|
||||
- SQL 先过守卫(SQLite 读法),只认 SELECT 和 WITH;表落完之后连接设成 query_only,再挂授权回调,只放行读表、
|
||||
调函数、递归公用表表达式。守卫是第一道,后两道不赌守卫没有漏洞;
|
||||
- 结果的行数、字节上限和查询层相同(QueryLimits),超时按查询层的缺省时限由进度回调中断;
|
||||
- 任一输入被截断 → 拒绝合并:拿只有前若干行的结果按键合并,缺的行不报错,只会悄悄少算;
|
||||
- 合并键两边类型不一致(文本 '001' 对数值 1)→ 警告。SQLite 比较时按亲和性隐式转换,'001'、'01' 都会等于 1;
|
||||
- 结果行数多于行数最多的输入 → 警告合并键可能不唯一(多对多匹配把行放大了)。
|
||||
|
||||
**逐格来历:宁可不下钻,也不下钻错。** 报告里引用合并结果的一格,证据面板要能指回它来自哪个输入的哪一格。只有一种
|
||||
写法认得出来:单层 SELECT,FROM 后面是输入表(可加别名、可 JOIN),选取项是某张输入表的列(可改名);没有分组、
|
||||
聚合、DISTINCT、子查询、公用表表达式、集合运算、窗口和 NATURAL JOIN。认出来以后,在原 SQL 的选取列表末尾补上每张
|
||||
输入表的 rowid 再执行一次:补列的结果去掉补的几列,必须和原结果逐行逐值相同,才把 rowid 记成来历;对不上整份不记。
|
||||
另外两种情况也不给:结果里一模一样的两行(说不清哪一行来自哪一行输入),以及来历那一格和结果的值对不上的列。
|
||||
表达式、聚合出来的列没有逐格来历,只有表级来历:来自哪几个输入、哪条合并 SQL。
|
||||
|
||||
纯函数:不连数据库、不读写工件、不发事件。取上游快照、存快照、发事件在执行器(engine/nodes/merge.py)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from app.data import guard
|
||||
from app.data.engine import _jsonable, column_types, open_memory_sqlite
|
||||
from app.data.guard import QueryLimits, SqlRejected
|
||||
from app.data.names import name_key
|
||||
from app.engine.direct_select import _kw, _lex, _Lex
|
||||
from app.engine.expressions import same_value
|
||||
|
||||
#: 合并查询结果的查询快照里 source 记这个。数据源名只能是小写英文、数字和下划线(api/datasources.py 的
|
||||
#: _NAME_PATTERN),永远撞不上它;证据面板、裁判摘录据此认出合并结果,不去按数据源名找遮罩
|
||||
MERGE_SOURCE = "合并查询"
|
||||
#: 一个合并节点最多几个输入。再多通常说明该在源库里先合并
|
||||
MAX_INPUTS = 8
|
||||
#: 别名(也就是内存库里的表名)最长几个字符
|
||||
ALIAS_MAX = 32
|
||||
|
||||
_ALIAS = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
#: SQLite 的全部关键字(sqlite.org/lang_keywords.html),加上 true / false。用作表名要加引号才读得对,
|
||||
#: 模型写合并 SQL 时多半不加——不如在起名时就挡住
|
||||
_SQLITE_KEYWORDS = frozenset("""
|
||||
abort action add after all alter always analyze and as asc attach autoincrement before begin between by cascade case
|
||||
cast check collate column commit conflict constraint create cross current current_date current_time current_timestamp
|
||||
database default deferrable deferred delete desc detach distinct do drop each else end escape except exclude exclusive
|
||||
exists explain fail filter first following for foreign from full generated glob group groups having if ignore immediate
|
||||
in index indexed initially inner insert instead intersect into is isnull join key last left like limit match
|
||||
materialized natural no not nothing notnull null nulls of offset on or order others outer over partition plan pragma
|
||||
preceding primary query raise range recursive references regexp reindex release rename replace restrict returning
|
||||
right rollback row rows savepoint select set table temp temporary then ties to transaction trigger unbounded union
|
||||
unique update using vacuum values view virtual when where window with without true false
|
||||
""".split())
|
||||
|
||||
#: 快照里记的列类型 → 内存表的列亲和性。number 用 NUMERIC:数据源把 DECIMAL 落成的文本("3200.50")进来就是数,
|
||||
#: 能直接参与运算;文本、日期一律 TEXT,'007' 的前导零不丢。拿不准的列不写类型,值原样存
|
||||
_AFFINITY = {"number": "NUMERIC", "boolean": "INTEGER", "text": "TEXT", "date": "TEXT", "datetime": "TEXT",
|
||||
"time": "TEXT"}
|
||||
_KIND_LABEL = {"number": "数值", "boolean": "布尔值", "text": "文本", "date": "日期", "datetime": "日期时间",
|
||||
"time": "时间"}
|
||||
#: SQLite 表的 rowid 有三个名字;输入里恰好有同名的列时换下一个,三个都占了就不追逐格来历
|
||||
_ROWID_NAMES = ("rowid", "_rowid_", "oid")
|
||||
#: 补进选取列表的来历列。带引号写,和输入的列名撞不上(输入的列名里不会有这个前缀,撞了也只是对不上、不记来历)
|
||||
_LINEAGE_COLUMN = "__agentlab_merge_row_{}"
|
||||
|
||||
SELECT_ONLY = "合并 SQL 只能是 SELECT 或 WITH 查询"
|
||||
_TEMPLATE = re.compile(r"\{\{.*?\}\}", re.S)
|
||||
|
||||
#: 授权回调放行的动作:读表、调函数、递归公用表表达式。别的一律拒(ATTACH、PRAGMA、写)
|
||||
_ALLOWED_ACTIONS = frozenset({sqlite3.SQLITE_SELECT, sqlite3.SQLITE_READ, sqlite3.SQLITE_FUNCTION,
|
||||
sqlite3.SQLITE_RECURSIVE})
|
||||
|
||||
|
||||
class MergeError(ValueError):
|
||||
"""合并做不成。消息直接给人看:哪个输入、为什么、怎么改。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MergeInput:
|
||||
"""一个输入:别名(内存库里的表名)、上游节点、它的查询快照。"""
|
||||
|
||||
alias: str
|
||||
node_id: str
|
||||
#: 上游查询快照的工件 id(tool 节点的 query_snapshot,或另一个合并节点的结果快照)
|
||||
artifact: str
|
||||
snapshot: dict[str, Any]
|
||||
#: 上游节点在画布上的名字,只用在报错里
|
||||
label: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Trace:
|
||||
"""合并结果的一格来自哪个输入的哪一格(只走一跳;输入本身是合并结果时由调用方再追)。"""
|
||||
|
||||
alias: str
|
||||
node_id: str | None
|
||||
artifact: str | None
|
||||
row: int
|
||||
column: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class MergeOutcome:
|
||||
columns: list[str]
|
||||
rows: list[list[Any]]
|
||||
truncated: bool
|
||||
#: 守卫规范过的那一条 SQL,执行的就是它
|
||||
sql: str
|
||||
column_types: dict[str, str]
|
||||
warnings: list[dict[str, Any]]
|
||||
#: 逐格来历(见模块说明);认不出写法、核对不上时为 None
|
||||
lineage: dict[str, Any] | None
|
||||
inputs: list[dict[str, Any]]
|
||||
#: 底层数据源(输入本身是合并结果的,展开成它的数据源),去重保序
|
||||
sources: list[str]
|
||||
mask_columns: list[str] = field(default_factory=list)
|
||||
|
||||
def snapshot(self) -> dict[str, Any]:
|
||||
"""存进工件库的查询快照:和数据源查询同形(columns、rows、row_count、truncated、sql、column_types、source),
|
||||
另记 inputs、sources、warnings、lineage。不含耗时之类每次不同的东西:同样的输入、同样的 SQL 得到同一个工件,
|
||||
节点重放、继续运行时快照 id 不变。"""
|
||||
out: dict[str, Any] = {
|
||||
"columns": self.columns, "rows": self.rows, "row_count": len(self.rows), "truncated": self.truncated,
|
||||
"sql": self.sql, "source": MERGE_SOURCE, "inputs": self.inputs, "sources": self.sources,
|
||||
"warnings": self.warnings,
|
||||
}
|
||||
if self.column_types:
|
||||
out["column_types"] = self.column_types
|
||||
if self.lineage is not None:
|
||||
out["lineage"] = self.lineage
|
||||
if self.mask_columns:
|
||||
out["mask_columns"] = self.mask_columns
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 静态检查:别名、SQL(画图时校验用,执行时同一套规则)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def alias_problem(alias: Any) -> str | None:
|
||||
"""别名能不能当内存库里的表名。能用返回 None。"""
|
||||
if not isinstance(alias, str) or not alias:
|
||||
return "别名不能为空"
|
||||
if len(alias) > ALIAS_MAX:
|
||||
return f"别名「{alias[:ALIAS_MAX]}…」太长,最多 {ALIAS_MAX} 个字符"
|
||||
if not _ALIAS.match(alias):
|
||||
return f"别名「{alias}」不是合法的表名:只能用英文字母、数字和下划线,且不能以数字开头"
|
||||
if alias.lower() in _SQLITE_KEYWORDS:
|
||||
return f"别名「{alias}」是 SQL 关键字,用作表名必须加引号,容易写错。请换一个,例如 sales、visits"
|
||||
if alias.lower().startswith("sqlite_"):
|
||||
return f"别名「{alias}」以 sqlite_ 开头,这是 SQLite 保留给内部表的名字,请换一个"
|
||||
return None
|
||||
|
||||
|
||||
def sql_problem(sql: Any) -> str | None:
|
||||
"""合并 SQL 有什么问题,没问题返回 None。{{ }} 运行时才有值,先按一个空值代进去检查写法。"""
|
||||
if not isinstance(sql, str) or not sql.strip():
|
||||
return "还没有填写合并 SQL"
|
||||
try:
|
||||
_checked(_TEMPLATE.sub("NULL", sql))
|
||||
except MergeError as e:
|
||||
return str(e)
|
||||
return None
|
||||
|
||||
|
||||
def _checked(sql: str) -> str:
|
||||
"""守卫(SQLite 读法)规范过的那一条语句。只认 SELECT / WITH;守卫的原话写给数据源的,换成合并查询自己的说法。"""
|
||||
if not sql or not sql.strip():
|
||||
raise MergeError("还没有填写合并 SQL")
|
||||
statements = guard.split_statements(sql, dialect="sqlite")
|
||||
if len(statements) > 1:
|
||||
raise MergeError(f"合并 SQL 只能写一条语句,这里有 {len(statements)} 条。请把它们合成一条 SELECT(可以用 WITH)")
|
||||
try:
|
||||
stmt = guard.check(sql, readonly=True, dialect="sqlite")
|
||||
except SqlRejected as e:
|
||||
# 只读拒绝的原话是「请在数据页另建一个可写的数据源」,合并查询用不上;结构变更(ATTACH、DROP)同理
|
||||
if guard.is_write(sql, dialect="sqlite"):
|
||||
raise MergeError(f"{SELECT_ONLY},不能写入、建表或连接别的库") from None
|
||||
raise MergeError(f"合并 SQL 无法执行:{e}") from None
|
||||
verb = guard.first_verb(stmt, dialect="sqlite")
|
||||
if verb not in ("select", "with"):
|
||||
raise MergeError(f"{SELECT_ONLY},当前是 {verb.upper() or '无法识别的语句'}")
|
||||
return stmt
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 执行
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def execute(inputs: list[MergeInput], sql: str, *, limits: QueryLimits | None = None) -> MergeOutcome:
|
||||
"""把输入落进内存库,执行合并 SQL。做不成抛 MergeError(消息给人看)。"""
|
||||
limits = limits or QueryLimits()
|
||||
if not inputs:
|
||||
raise MergeError("还没有配置输入:至少选择一个上游的查询节点")
|
||||
if len(inputs) > MAX_INPUTS:
|
||||
raise MergeError(f"输入最多 {MAX_INPUTS} 个,这里有 {len(inputs)} 个。请先在源库里合并一部分")
|
||||
_check_aliases(inputs)
|
||||
tables = {inp.alias: _input_columns(inp) for inp in inputs}
|
||||
stmt = _checked(sql)
|
||||
|
||||
conn = open_memory_sqlite()
|
||||
try:
|
||||
rowids = {inp.alias: _load(conn, inp, tables[inp.alias]) for inp in inputs}
|
||||
# 表落完就只读:授权回调之外的又一道(守卫、授权回调任何一处有漏洞,写也写不进去)
|
||||
conn.execute("PRAGMA query_only = 1")
|
||||
conn.set_authorizer(_authorize)
|
||||
deadline = time.monotonic() + float(limits.timeout_seconds)
|
||||
conn.set_progress_handler(lambda: 1 if time.monotonic() > deadline else 0, 1000)
|
||||
columns, rows, truncated = _read(conn, stmt, limits)
|
||||
plan = _lineage_plan(stmt, inputs, tables, rowids)
|
||||
lineage = _lineage(conn, plan, columns, rows, inputs, tables) if plan is not None else None
|
||||
except sqlite3.Error as e:
|
||||
raise MergeError(_sqlite_reason(e, limits)) from None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
warnings = _warnings(stmt, inputs, tables, rows, truncated, limits)
|
||||
return MergeOutcome(
|
||||
columns=columns, rows=rows, truncated=truncated, sql=stmt, column_types=column_types(columns, rows),
|
||||
warnings=warnings, lineage=lineage, inputs=[_summary(inp, tables[inp.alias]) for inp in inputs],
|
||||
sources=_sources(inputs), mask_columns=_masks(inputs, columns, plan),
|
||||
)
|
||||
|
||||
|
||||
def _check_aliases(inputs: list[MergeInput]) -> None:
|
||||
seen: dict[str, str] = {}
|
||||
for inp in inputs:
|
||||
if problem := alias_problem(inp.alias):
|
||||
raise MergeError(problem)
|
||||
key = inp.alias.lower()
|
||||
if key in seen:
|
||||
raise MergeError(f"别名「{seen[key]}」和「{inp.alias}」只差大小写:SQLite 的表名不分大小写,会被当成同一张表。"
|
||||
"请换一个别名")
|
||||
seen[key] = inp.alias
|
||||
|
||||
|
||||
def _who(inp: MergeInput) -> str:
|
||||
return f"输入「{inp.alias}」" + (f"({inp.label})" if inp.label and inp.label != inp.alias else "")
|
||||
|
||||
|
||||
def _input_columns(inp: MergeInput) -> list[str]:
|
||||
snapshot = inp.snapshot
|
||||
if not isinstance(snapshot, dict) or not isinstance(snapshot.get("columns"), list) \
|
||||
or not isinstance(snapshot.get("rows"), list):
|
||||
raise MergeError(f"{_who(inp)}不是查询结果:快照里没有列和行")
|
||||
if snapshot.get("truncated"):
|
||||
raise MergeError(
|
||||
f"{_who(inp)}的查询结果被截断,只拿到了前 {len(snapshot['rows'])} 行。拿不完整的结果按键合并,缺的行不会报错,"
|
||||
"只会让合并结果悄悄少算。请在源库里先聚合到合并需要的粒度(例如按日期和门店 GROUP BY),"
|
||||
"或者加条件缩小范围,让结果不超过查询上限")
|
||||
columns = [str(c) for c in snapshot["columns"]]
|
||||
if not columns:
|
||||
raise MergeError(f"{_who(inp)}的查询结果没有列")
|
||||
seen: dict[str, str] = {}
|
||||
for name in columns:
|
||||
key = name_key(name)
|
||||
if key in seen:
|
||||
raise MergeError(f"{_who(inp)}的结果里有重名的列「{name}」:SQLite 的列名不分大小写,同名的列放不进一张表。"
|
||||
"请在源查询中用 AS 给它们起不同的名字")
|
||||
seen[key] = name
|
||||
return columns
|
||||
|
||||
|
||||
def _quote(name: str) -> str:
|
||||
return '"' + name.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def _kind(snapshot: dict[str, Any], column: str) -> str | None:
|
||||
"""这一列的类型:查询时记下的(column_types)优先;老快照没记的按值猜,猜不准不说。"""
|
||||
types = snapshot.get("column_types")
|
||||
if isinstance(types, dict) and isinstance(types.get(column), str):
|
||||
return types[column]
|
||||
columns = [str(c) for c in snapshot.get("columns") or []]
|
||||
if column not in columns:
|
||||
return None
|
||||
at = columns.index(column)
|
||||
kinds = {_value_kind(row[at]) for row in snapshot.get("rows") or []
|
||||
if isinstance(row, list) and at < len(row) and row[at] is not None}
|
||||
return next(iter(kinds)) if len(kinds) == 1 and None not in kinds else None
|
||||
|
||||
|
||||
def _value_kind(value: Any) -> str | None:
|
||||
if isinstance(value, bool):
|
||||
return "boolean"
|
||||
if isinstance(value, (int, float)):
|
||||
return "number"
|
||||
return "text" if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _cell_in(value: Any) -> Any:
|
||||
"""快照里的值 → 写进内存库的值。快照是 JSON:布尔写成 0 / 1,对象和数组写成 JSON 文本。"""
|
||||
if value is None or isinstance(value, (str, int, float)) and not isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, bool):
|
||||
return int(value)
|
||||
if isinstance(value, (dict, list)):
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||
return str(value)
|
||||
|
||||
|
||||
def _load(conn: sqlite3.Connection, inp: MergeInput, columns: list[str]) -> str | None:
|
||||
"""建表、按快照的顺序写入行,返回这张表的 rowid 用哪个名字(第 i 行的 rowid 是 i + 1)。"""
|
||||
snapshot = inp.snapshot
|
||||
decl = ", ".join(f"{_quote(c)} {_AFFINITY.get(_kind(snapshot, c) or '', '')}".rstrip() for c in columns)
|
||||
conn.execute(f"CREATE TABLE {_quote(inp.alias)} ({decl})")
|
||||
taken = {c.lower() for c in columns}
|
||||
rowid = next((n for n in _ROWID_NAMES if n not in taken), None)
|
||||
width = len(columns)
|
||||
rows: list[list[Any]] = []
|
||||
for i, row in enumerate(snapshot["rows"]):
|
||||
if isinstance(row, dict):
|
||||
row = [row.get(c) for c in columns]
|
||||
if not isinstance(row, list) or len(row) != width:
|
||||
raise MergeError(f"{_who(inp)}的快照第 {i} 行和列数对不上,快照可能已损坏")
|
||||
rows.append([i + 1, *map(_cell_in, row)] if rowid else [_cell_in(v) for v in row])
|
||||
names = ", ".join([*([rowid] if rowid else []), *map(_quote, columns)])
|
||||
marks = ", ".join("?" * (width + (1 if rowid else 0)))
|
||||
conn.executemany(f"INSERT INTO {_quote(inp.alias)} ({names}) VALUES ({marks})", rows)
|
||||
return rowid
|
||||
|
||||
|
||||
def _authorize(action: int, *_args: Any) -> int:
|
||||
return sqlite3.SQLITE_OK if action in _ALLOWED_ACTIONS else sqlite3.SQLITE_DENY
|
||||
|
||||
|
||||
def _read(conn: sqlite3.Connection, stmt: str, limits: QueryLimits) -> tuple[list[str], list[list[Any]], bool]:
|
||||
"""逐行读、边读边判上限:和查询层(data.engine.run_query)同一个口径,到了行数或字节上限就停,记 truncated。"""
|
||||
cursor = conn.execute(stmt)
|
||||
columns = [str(d[0]) for d in cursor.description or []]
|
||||
if not columns:
|
||||
raise MergeError("合并 SQL 没有返回任何列")
|
||||
rows: list[list[Any]] = []
|
||||
size = 0
|
||||
truncated = False
|
||||
for row in cursor:
|
||||
values = [_jsonable(v) for v in row]
|
||||
rows.append(values)
|
||||
size += len(json.dumps(values, ensure_ascii=False, default=str))
|
||||
if len(rows) >= limits.max_rows or size >= limits.max_bytes:
|
||||
truncated = True
|
||||
break
|
||||
return columns, rows, truncated
|
||||
|
||||
|
||||
def _sqlite_reason(e: sqlite3.Error, limits: QueryLimits) -> str:
|
||||
text = str(e)
|
||||
if "interrupted" in text:
|
||||
seconds = float(limits.timeout_seconds)
|
||||
shown = f"{seconds:g}"
|
||||
return f"合并超过 {shown} 秒被中断。请在源查询里先聚合,或在合并 SQL 中加条件缩小范围"
|
||||
if "not authorized" in text or "readonly" in text:
|
||||
return f"{SELECT_ONLY},只能读取输入表"
|
||||
# SQLite 的原话(no such column: x、ambiguous column name: 日期)正是改 SQL 要的线索,原样带上
|
||||
return f"合并 SQL 执行失败:{text}"
|
||||
|
||||
|
||||
def _summary(inp: MergeInput, columns: list[str]) -> dict[str, Any]:
|
||||
source = inp.snapshot.get("source")
|
||||
return {"alias": inp.alias, "node_id": inp.node_id, "artifact": inp.artifact, "rows": len(inp.snapshot["rows"]),
|
||||
"columns": columns, "source": str(source) if source else None}
|
||||
|
||||
|
||||
def _sources(inputs: list[MergeInput]) -> list[str]:
|
||||
out: list[str] = []
|
||||
for inp in inputs:
|
||||
nested = inp.snapshot.get("sources")
|
||||
names = nested if inp.snapshot.get("source") == MERGE_SOURCE and isinstance(nested, list) \
|
||||
else [inp.snapshot.get("source")]
|
||||
out.extend(str(n) for n in names if n and str(n) not in out)
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 逐格来历
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: 整条语句里出现就不追来历的词:子查询、集合运算、公用表表达式、分组、窗口、去重、NATURAL JOIN
|
||||
_NO_LINEAGE = frozenset({"select", "union", "intersect", "except", "with", "recursive", "group", "having", "window",
|
||||
"over", "distinct", "values", "natural", "filter"})
|
||||
#: 聚合函数(名字后面紧跟「(」)。没有 GROUP BY 的聚合把整张表压成一行,选取项里的裸列取的是哪一行说不准
|
||||
_AGGREGATES = frozenset({"count", "sum", "avg", "min", "max", "total", "group_concat", "string_agg", "median",
|
||||
"json_group_array", "json_group_object", "jsonb_group_array", "jsonb_group_object"})
|
||||
#: 不加引号的字面量关键字:选取项写 NULL、CURRENT_DATE 不是列
|
||||
_LITERALS = frozenset({"null", "true", "false", "current_date", "current_time", "current_timestamp"})
|
||||
#: 来源表后面紧跟这些词时,它们不是省略了 AS 的表别名
|
||||
_AFTER_SOURCE = frozenset({"on", "using", "where", "order", "limit", "join", "left", "right", "full", "inner", "cross",
|
||||
"outer", "indexed", "not", "natural", "group", "having", "window", "union", "intersect",
|
||||
"except", "as"})
|
||||
_JOIN_WORDS = frozenset({"join", "left", "right", "full", "inner", "cross", "natural"})
|
||||
_TAIL = frozenset({"where", "order", "limit"})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Source:
|
||||
#: 输入别名(内存库里的表名)
|
||||
alias: str
|
||||
#: SQL 里怎么称呼它:写了表别名就是表别名,否则是表名本身
|
||||
ref: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Plan:
|
||||
sources: list[_Source]
|
||||
#: 第 i 个结果列:(第几个来源, 输入里的列名);表达式、说不准的列是 None
|
||||
columns: list[tuple[int, str] | None]
|
||||
#: 在原 SQL 的哪个位置补来历列(顶层 FROM 前面)
|
||||
insert_at: int
|
||||
stmt: str
|
||||
rowids: list[str]
|
||||
|
||||
|
||||
def _punct(lex: _Lex | None, text: str) -> bool:
|
||||
return lex is not None and lex.kind == "punct" and lex.text == text
|
||||
|
||||
|
||||
def _named(lex: _Lex | None) -> bool:
|
||||
return lex is not None and lex.kind in ("word", "name")
|
||||
|
||||
|
||||
def _find(names: list[str], wanted: str) -> str | None:
|
||||
key = name_key(wanted)
|
||||
return next((n for n in names if name_key(n) == key), None)
|
||||
|
||||
|
||||
def _lineage_plan(stmt: str, inputs: list[MergeInput], tables: dict[str, list[str]],
|
||||
rowids: dict[str, str | None]) -> _Plan | None:
|
||||
"""认出「单层 SELECT、选取项是输入表的列」这种写法;认不出返回 None(整份不记来历)。"""
|
||||
toks = _lex(stmt)
|
||||
if not toks or _kw(toks[0]) != "select":
|
||||
return None
|
||||
if any(_kw(t) in _NO_LINEAGE for t in toks[1:]):
|
||||
return None
|
||||
if any(_kw(t) in _AGGREGATES and _punct(nxt, "(") for t, nxt in zip(toks, toks[1:])):
|
||||
return None
|
||||
depth, from_at = 0, None
|
||||
for i, t in enumerate(toks):
|
||||
if _punct(t, "("):
|
||||
depth += 1
|
||||
elif _punct(t, ")"):
|
||||
depth -= 1
|
||||
elif depth == 0 and _kw(t) == "from":
|
||||
from_at = i
|
||||
break
|
||||
if from_at is None:
|
||||
return None
|
||||
start = 2 if _kw(toks[1]) == "all" else 1
|
||||
parsed = _from_clause(toks, from_at + 1, tables)
|
||||
if parsed is None:
|
||||
return None
|
||||
sources, using = parsed
|
||||
columns: list[tuple[int, str] | None] = []
|
||||
for item in _split_items(toks[start:from_at]):
|
||||
mapped = _item(item, sources, tables, using)
|
||||
if mapped is None:
|
||||
return None
|
||||
columns.extend(mapped)
|
||||
if any(rowids.get(s.alias) is None for s in sources):
|
||||
return None
|
||||
frm = toks[from_at]
|
||||
return _Plan(sources=sources, columns=columns, insert_at=frm.end - len(frm.text), stmt=stmt,
|
||||
rowids=[str(rowids[s.alias]) for s in sources])
|
||||
|
||||
|
||||
def _split_items(toks: list[_Lex]) -> list[list[_Lex]]:
|
||||
items: list[list[_Lex]] = [[]]
|
||||
depth = 0
|
||||
for t in toks:
|
||||
if _punct(t, "("):
|
||||
depth += 1
|
||||
elif _punct(t, ")"):
|
||||
depth -= 1
|
||||
if depth == 0 and _punct(t, ","):
|
||||
items.append([])
|
||||
else:
|
||||
items[-1].append(t)
|
||||
return items
|
||||
|
||||
|
||||
def _from_clause(toks: list[_Lex], i: int, tables: dict[str, list[str]]) -> tuple[list[_Source], set[str]] | None:
|
||||
"""FROM 之后:来源表 [AS 别名],用逗号或 JOIN 连接,JOIN 后面可跟 ON 条件或 USING (列…);之后只许
|
||||
WHERE / ORDER BY / LIMIT。来源只能是输入表(不认子查询、表函数、带库名的表)。"""
|
||||
sources: list[_Source] = []
|
||||
using: set[str] = set()
|
||||
n = len(toks)
|
||||
while True:
|
||||
t = toks[i] if i < n else None
|
||||
if not _named(t) or (t.kind == "word" and name_key(t.text) in _SQLITE_KEYWORDS):
|
||||
return None
|
||||
table = _find(list(tables), t.text)
|
||||
if table is None or _punct(toks[i + 1] if i + 1 < n else None, ".") \
|
||||
or _punct(toks[i + 1] if i + 1 < n else None, "("):
|
||||
return None
|
||||
ref, i = t.text, i + 1
|
||||
nxt = toks[i] if i < n else None
|
||||
if _kw(nxt) == "as":
|
||||
if not _named(toks[i + 1] if i + 1 < n else None):
|
||||
return None
|
||||
ref, i = toks[i + 1].text, i + 2
|
||||
elif _named(nxt) and (nxt.kind == "name" or name_key(nxt.text) not in _AFTER_SOURCE):
|
||||
ref, i = nxt.text, i + 1
|
||||
sources.append(_Source(alias=table, ref=ref))
|
||||
word = _kw(toks[i]) if i < n else None
|
||||
if word == "on":
|
||||
depth, i = 0, i + 1
|
||||
while i < n:
|
||||
t = toks[i]
|
||||
if _punct(t, "("):
|
||||
depth += 1
|
||||
elif _punct(t, ")"):
|
||||
depth -= 1
|
||||
elif depth == 0 and (_punct(t, ",") or _kw(t) in _JOIN_WORDS or _kw(t) in _TAIL):
|
||||
break
|
||||
i += 1
|
||||
elif word == "using":
|
||||
if not _punct(toks[i + 1] if i + 1 < n else None, "("):
|
||||
return None
|
||||
i += 2
|
||||
while i < n and not _punct(toks[i], ")"):
|
||||
if _named(toks[i]):
|
||||
using.add(name_key(toks[i].text))
|
||||
elif not _punct(toks[i], ","):
|
||||
return None
|
||||
i += 1
|
||||
i += 1
|
||||
if i >= n:
|
||||
return sources, using
|
||||
t = toks[i]
|
||||
if _punct(t, ","):
|
||||
i += 1
|
||||
continue
|
||||
word = _kw(t)
|
||||
if word in _TAIL:
|
||||
return sources, using
|
||||
if word not in ("join", "left", "right", "full", "inner", "cross"):
|
||||
return None
|
||||
while i < n and _kw(toks[i]) in ("left", "right", "full", "inner", "cross", "outer"):
|
||||
i += 1
|
||||
if i >= n or _kw(toks[i]) != "join":
|
||||
return None
|
||||
i += 1
|
||||
|
||||
|
||||
def _source_of(ref: str, sources: list[_Source]) -> int | None:
|
||||
key = name_key(ref)
|
||||
hits = [k for k, s in enumerate(sources) if name_key(s.ref) == key]
|
||||
return hits[0] if len(hits) == 1 else None
|
||||
|
||||
|
||||
def _item(item: list[_Lex], sources: list[_Source], tables: dict[str, list[str]],
|
||||
using: set[str]) -> list[tuple[int, str] | None] | None:
|
||||
"""一个选取项展开成几个结果列的来历。返回 None 表示整份不追(* 遇上 USING:SQLite 只出一份合并掉的列)。"""
|
||||
if len(item) == 1 and _punct(item[0], "*"):
|
||||
if using:
|
||||
return None
|
||||
return [(k, c) for k, s in enumerate(sources) for c in tables[s.alias]]
|
||||
if len(item) == 3 and _named(item[0]) and _punct(item[1], ".") and _punct(item[2], "*"):
|
||||
k = _source_of(item[0].text, sources)
|
||||
return None if k is None else [(k, c) for c in tables[sources[k].alias]]
|
||||
body = item
|
||||
if len(item) >= 3 and _kw(item[-2]) == "as" and _named(item[-1]):
|
||||
body = item[:-2]
|
||||
elif len(item) >= 2 and _named(item[-1]) and _named(item[-2]) \
|
||||
and not (item[-1].kind == "word" and name_key(item[-1].text) in _SQLITE_KEYWORDS):
|
||||
body = item[:-1]
|
||||
if len(body) == 1 and _named(body[0]):
|
||||
word = body[0]
|
||||
if word.kind == "word" and name_key(word.text) in _LITERALS:
|
||||
return [None]
|
||||
owners = [(k, col) for k, s in enumerate(sources) if (col := _find(tables[s.alias], word.text)) is not None]
|
||||
if len(owners) != 1 or name_key(word.text) in using:
|
||||
return [None]
|
||||
return [owners[0]]
|
||||
if len(body) == 3 and _named(body[0]) and _punct(body[1], ".") and _named(body[2]):
|
||||
k = _source_of(body[0].text, sources)
|
||||
col = _find(tables[sources[k].alias], body[2].text) if k is not None else None
|
||||
return [(k, col) if k is not None and col is not None else None]
|
||||
return [None]
|
||||
|
||||
|
||||
def _hashable(row: list[Any]) -> str:
|
||||
return json.dumps(row, ensure_ascii=False, sort_keys=True, default=str)
|
||||
|
||||
|
||||
def _lineage(conn: sqlite3.Connection, plan: _Plan, columns: list[str], rows: list[list[Any]],
|
||||
inputs: list[MergeInput], tables: dict[str, list[str]]) -> dict[str, Any] | None:
|
||||
"""补上 rowid 再执行一次,逐行逐值核对,得出每一格来自哪个输入的哪一行。核对不上返回 None。"""
|
||||
width = len(columns)
|
||||
if len(plan.columns) != width or not any(plan.columns):
|
||||
return None
|
||||
extra = ", ".join(f"{_quote(s.ref)}.{rowid} AS {_quote(_LINEAGE_COLUMN.format(k))}"
|
||||
for k, (s, rowid) in enumerate(zip(plan.sources, plan.rowids)))
|
||||
head = plan.stmt[:plan.insert_at].rstrip()
|
||||
try:
|
||||
augmented = _checked(f"{head}, {extra} {plan.stmt[plan.insert_at:]}")
|
||||
cursor = conn.execute(augmented)
|
||||
except (MergeError, sqlite3.Error):
|
||||
return None
|
||||
got: list[tuple[Any, ...]] = []
|
||||
for row in cursor:
|
||||
if len(got) >= len(rows):
|
||||
break
|
||||
got.append(row)
|
||||
if len(got) != len(rows) or any(len(g) != width + len(plan.sources) for g in got):
|
||||
return None
|
||||
if any([_jsonable(v) for v in g[:width]] != r for g, r in zip(got, rows)):
|
||||
return None
|
||||
origin: list[list[int | None]] = [[v - 1 if isinstance(v, int) and v >= 1 else None for v in g[width:]]
|
||||
for g in got]
|
||||
# 一模一样的两行结果:说不清哪一行来自哪一行输入
|
||||
counts = Counter(_hashable(r) for r in rows)
|
||||
for i, r in enumerate(rows):
|
||||
if counts[_hashable(r)] > 1:
|
||||
origin[i] = [None] * len(plan.sources)
|
||||
snaps = {inp.alias: inp.snapshot for inp in inputs}
|
||||
mapped: list[dict[str, Any] | None] = []
|
||||
for j, target in enumerate(plan.columns):
|
||||
if target is None:
|
||||
mapped.append(None)
|
||||
continue
|
||||
k, column = target
|
||||
snap = snaps[plan.sources[k].alias]
|
||||
at = tables[plan.sources[k].alias].index(column)
|
||||
ok = True
|
||||
for i, row in enumerate(rows):
|
||||
src = origin[i][k]
|
||||
if src is None:
|
||||
continue
|
||||
if not (0 <= src < len(snap["rows"])) or not _same(row[j], _row(snap["rows"][src], tables[plan.sources[k].alias])[at]):
|
||||
ok = False
|
||||
break
|
||||
mapped.append({"source": k, "column": column} if ok else None)
|
||||
if not any(mapped):
|
||||
return None
|
||||
return {"sources": [{"input": s.alias, "ref": s.ref} for s in plan.sources], "columns": mapped, "rows": origin}
|
||||
|
||||
|
||||
def _row(row: Any, columns: list[str]) -> list[Any]:
|
||||
return [row.get(c) for c in columns] if isinstance(row, dict) else row
|
||||
|
||||
|
||||
def _same(merged: Any, original: Any) -> bool:
|
||||
if merged is None or original is None:
|
||||
return merged is None and original is None
|
||||
return same_value(merged, _cell_in(original))
|
||||
|
||||
|
||||
def trace_cell(snapshot: Any, row: Any, column: Any) -> Trace | None:
|
||||
"""合并结果的这一格来自哪个输入的哪一格;没有逐格来历返回 None。列按第一次出现的位置认(同 locate_cell)。"""
|
||||
if not isinstance(snapshot, dict) or snapshot.get("source") != MERGE_SOURCE:
|
||||
return None
|
||||
lineage, columns = snapshot.get("lineage"), snapshot.get("columns")
|
||||
if not isinstance(lineage, dict) or not isinstance(columns, list) or column not in columns:
|
||||
return None
|
||||
if isinstance(row, bool) or not isinstance(row, int):
|
||||
return None
|
||||
mapped, origin, sources = lineage.get("columns"), lineage.get("rows"), lineage.get("sources")
|
||||
at = columns.index(column)
|
||||
if not (isinstance(mapped, list) and isinstance(origin, list) and isinstance(sources, list)
|
||||
and at < len(mapped) and 0 <= row < len(origin)):
|
||||
return None
|
||||
target = mapped[at]
|
||||
if not isinstance(target, dict) or not isinstance(target.get("source"), int):
|
||||
return None
|
||||
k = target["source"]
|
||||
line = origin[row]
|
||||
if not isinstance(line, list) or not 0 <= k < len(line) or not 0 <= k < len(sources):
|
||||
return None
|
||||
src = line[k]
|
||||
alias = (sources[k] or {}).get("input") if isinstance(sources[k], dict) else None
|
||||
if isinstance(src, bool) or not isinstance(src, int) or not isinstance(alias, str):
|
||||
return None
|
||||
entry = next((e for e in snapshot.get("inputs") or [] if isinstance(e, dict) and e.get("alias") == alias), None)
|
||||
if entry is None:
|
||||
return None
|
||||
return Trace(alias=alias, node_id=entry.get("node_id"), artifact=entry.get("artifact"), row=src,
|
||||
column=str(target.get("column")))
|
||||
|
||||
|
||||
def _masks(inputs: list[MergeInput], columns: list[str], plan: _Plan | None) -> list[str]:
|
||||
"""合并结果要遮罩的列:各输入当时记下的遮罩列,加上来自遮罩列、换了名字的结果列(宁可多遮)。"""
|
||||
hidden: dict[str, set[str]] = {}
|
||||
names: list[str] = []
|
||||
for inp in inputs:
|
||||
recorded = inp.snapshot.get("mask_columns")
|
||||
cols = {str(c).lower() for c in recorded} if isinstance(recorded, list) else set()
|
||||
hidden[inp.alias] = cols
|
||||
names.extend(str(c) for c in recorded or [] if isinstance(c, str))
|
||||
if plan is not None and len(plan.columns) == len(columns):
|
||||
for name, target in zip(columns, plan.columns):
|
||||
if target is not None and target[1].lower() in hidden.get(plan.sources[target[0]].alias, set()):
|
||||
names.append(name)
|
||||
return sorted(dict.fromkeys(names))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 警告:合并键类型、行数放大、结果截断
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _key_pairs(stmt: str, tables: dict[str, list[str]]) -> list[tuple[str, str, str, str]]:
|
||||
"""合并条件里的键:ON / WHERE 里「列 = 列」两边落在不同的输入表上,以及 USING (列)。只用来提示,认不全不要紧。"""
|
||||
toks = _lex(stmt, strict=False) or []
|
||||
n = len(toks)
|
||||
refs: dict[str, str] = {}
|
||||
pairs: list[tuple[str, str, str, str]] = []
|
||||
placed: list[str] = [] # 当前 FROM 列表里已经出现过的输入表(USING 找左边那张)
|
||||
i = 0
|
||||
while i < n:
|
||||
word = _kw(toks[i])
|
||||
if word in ("from", "join") or (_punct(toks[i], ",") and placed and i > 0 and _named(toks[i - 1])):
|
||||
if word == "from":
|
||||
placed = []
|
||||
j = i + 1
|
||||
t = toks[j] if j < n else None
|
||||
table = _find(list(tables), t.text) if _named(t) else None
|
||||
if table is not None and not _punct(toks[j + 1] if j + 1 < n else None, "."):
|
||||
ref, j = t.text, j + 1
|
||||
nxt = toks[j] if j < n else None
|
||||
if _kw(nxt) == "as" and _named(toks[j + 1] if j + 1 < n else None):
|
||||
ref, j = toks[j + 1].text, j + 2
|
||||
elif _named(nxt) and (nxt.kind == "name" or name_key(nxt.text) not in _AFTER_SOURCE):
|
||||
ref, j = nxt.text, j + 1
|
||||
refs[name_key(ref)] = table
|
||||
refs.setdefault(name_key(table), table)
|
||||
if _kw(toks[j] if j < n else None) == "using" and _punct(toks[j + 1] if j + 1 < n else None, "("):
|
||||
k = j + 2
|
||||
while k < n and not _punct(toks[k], ")"):
|
||||
if _named(toks[k]):
|
||||
col = _find(tables[table], toks[k].text)
|
||||
left = next((p for p in reversed(placed) if _find(tables[p], toks[k].text)), None)
|
||||
if col is not None and left is not None:
|
||||
pairs.append((left, str(_find(tables[left], toks[k].text)), table, col))
|
||||
k += 1
|
||||
placed.append(table)
|
||||
i = j
|
||||
continue
|
||||
i += 1
|
||||
|
||||
def resolve(at: int, step: int) -> tuple[tuple[str, str] | None, int]:
|
||||
"""从 at 开始(step=1 往右、-1 往左)读一个列引用:(表, 列)、它占了几个记号。"""
|
||||
if step > 0:
|
||||
seq = toks[at:at + 3]
|
||||
else:
|
||||
seq = toks[max(0, at - 2):at + 1]
|
||||
if len(seq) == 3 and _named(seq[0]) and _punct(seq[1], ".") and _named(seq[2]):
|
||||
table = refs.get(name_key(seq[0].text))
|
||||
col = _find(tables[table], seq[2].text) if table else None
|
||||
return ((table, col) if table and col else None), 3
|
||||
one = toks[at] if 0 <= at < n else None
|
||||
if _named(one):
|
||||
owners = [(t, c) for t in tables if (c := _find(tables[t], one.text)) is not None]
|
||||
return (owners[0] if len(owners) == 1 else None), 1
|
||||
return None, 1
|
||||
|
||||
for i, t in enumerate(toks):
|
||||
if not _punct(t, "="):
|
||||
continue
|
||||
prev = toks[i - 1] if i > 0 else None
|
||||
# 「==」的第二个等号、<=、>=、!= 都不是比较的开头(分词把每个标点切成一个记号)
|
||||
if prev is None or (prev.kind == "punct" and prev.text in "=<>!"):
|
||||
continue
|
||||
right_at = i + 2 if _punct(toks[i + 1] if i + 1 < n else None, "=") else i + 1
|
||||
left, width = resolve(i - 1, -1)
|
||||
before = toks[i - 1 - width] if i - 1 - width >= 0 else None
|
||||
if before is not None and _punct(before, "."):
|
||||
left = None # a.b.c 这种带库名的写法:不是输入表的列
|
||||
right, rwidth = resolve(right_at, 1)
|
||||
after = toks[right_at + rwidth] if right_at + rwidth < n else None
|
||||
if after is not None and (_punct(after, ".") or _punct(after, "(")):
|
||||
right = None
|
||||
if left and right and left[0] != right[0]:
|
||||
pairs.append((left[0], left[1], right[0], right[1]))
|
||||
seen: set[frozenset[tuple[str, str]]] = set()
|
||||
out: list[tuple[str, str, str, str]] = []
|
||||
for a, ac, b, bc in pairs:
|
||||
key = frozenset({(a, ac), (b, bc)})
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
out.append((a, ac, b, bc))
|
||||
return out
|
||||
|
||||
|
||||
def _example(snapshot: dict[str, Any], column: str) -> str | None:
|
||||
columns = [str(c) for c in snapshot.get("columns") or []]
|
||||
if column not in columns:
|
||||
return None
|
||||
at = columns.index(column)
|
||||
for row in snapshot.get("rows") or []:
|
||||
value = row[at] if isinstance(row, list) and at < len(row) else None
|
||||
if value is not None:
|
||||
return f"'{value}'" if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
|
||||
return None
|
||||
|
||||
|
||||
def _clash(a: str | None, b: str | None) -> bool:
|
||||
"""两边的类型放在一起比会不会出错:数值对文本类,日期对日期时间('2026-05-01' 永远不等于 '2026-05-01T00:00:00')。"""
|
||||
if not a or not b or a == b:
|
||||
return False
|
||||
numeric = {"number", "boolean"}
|
||||
if (a in numeric) != (b in numeric):
|
||||
return True
|
||||
return {a, b} == {"date", "datetime"}
|
||||
|
||||
|
||||
def _warnings(stmt: str, inputs: list[MergeInput], tables: dict[str, list[str]], rows: list[list[Any]],
|
||||
truncated: bool, limits: QueryLimits) -> list[dict[str, Any]]:
|
||||
snaps = {inp.alias: inp.snapshot for inp in inputs}
|
||||
pairs = _key_pairs(stmt, tables)
|
||||
out: list[dict[str, Any]] = []
|
||||
for a, ac, b, bc in pairs:
|
||||
ka, kb = _kind(snaps[a], ac), _kind(snaps[b], bc)
|
||||
if not _clash(ka, kb):
|
||||
continue
|
||||
ea, eb = _example(snaps[a], ac), _example(snaps[b], bc)
|
||||
left = f"{a}.{ac} 是{_KIND_LABEL.get(str(ka), ka)}" + (f"(例如 {ea})" if ea else "")
|
||||
right = f"{b}.{bc} 是{_KIND_LABEL.get(str(kb), kb)}" + (f"(例如 {eb})" if eb else "")
|
||||
if {ka, kb} == {"date", "datetime"}:
|
||||
tail = ("日期和日期时间按文本比较永远不相等,这组键可能一行都匹配不上。请在源查询中统一格式,"
|
||||
"或在合并 SQL 中用 date() 转换后再比较")
|
||||
else:
|
||||
tail = ("SQLite 比较时会做隐式转换:文本形式的编号(如 '001'、'01')都会等于数值 1,也可能完全匹配不上。"
|
||||
"请在源查询中统一类型,或在合并 SQL 中用 CAST 明确转换")
|
||||
out.append({"code": "key_type_mismatch", "keys": [f"{a}.{ac}", f"{b}.{bc}"],
|
||||
"message": f"合并键类型不一致:{left},{right}。{tail}"})
|
||||
largest = max(inputs, key=lambda inp: len(inp.snapshot["rows"]))
|
||||
biggest = len(largest.snapshot["rows"])
|
||||
if len(rows) > biggest:
|
||||
keys: dict[str, list[str]] = {}
|
||||
for a, ac, b, bc in pairs:
|
||||
for alias, col in ((a, ac), (b, bc)):
|
||||
if col not in keys.setdefault(alias, []):
|
||||
keys[alias].append(col)
|
||||
dups = [_duplicates(alias, snaps[alias], cols) for alias, cols in keys.items()]
|
||||
detail = ";".join(d for d in dups if d)
|
||||
out.append({"code": "rows_grew", "message": (
|
||||
f"合并结果有 {len(rows)} 行,多于行数最多的输入「{largest.alias}」({biggest} 行):合并键可能不唯一,"
|
||||
"同一行被重复匹配。" + (f"{detail}。" if detail else "")
|
||||
+ "请检查合并条件是否覆盖了全部键(例如同时按日期和门店),或先在源库里聚合到相同粒度")})
|
||||
if truncated:
|
||||
out.append({"code": "result_truncated", "message": (
|
||||
f"合并结果超过上限({limits.max_rows} 行或约 {max(1, limits.max_bytes // 1_000_000)} MB),只保留了前 "
|
||||
f"{len(rows)} 行,下游拿到的不是完整结果。请在合并 SQL 中聚合或加条件缩小范围")})
|
||||
return out
|
||||
|
||||
|
||||
def _duplicates(alias: str, snapshot: dict[str, Any], cols: list[str]) -> str | None:
|
||||
"""这个输入按合并键有没有重复:有的话举第一组,说一共几组。"""
|
||||
columns = [str(c) for c in snapshot.get("columns") or []]
|
||||
if not cols or any(c not in columns for c in cols):
|
||||
return None
|
||||
at = [columns.index(c) for c in cols]
|
||||
counts = Counter(tuple(_hashable([row[j]]) for j in at) for row in snapshot.get("rows") or []
|
||||
if isinstance(row, list))
|
||||
repeated = [(key, n) for key, n in counts.items() if n > 1]
|
||||
if not repeated:
|
||||
return None
|
||||
key, n = repeated[0]
|
||||
shown = " / ".join(str(json.loads(v)[0]) for v in key)
|
||||
more = f"等 {len(repeated)} 组" if len(repeated) > 1 else ""
|
||||
return f"「{alias}」中按({'、'.join(cols)})有重复的键,例如 {shown} 出现 {n} 次{more}"
|
||||
@@ -0,0 +1,132 @@
|
||||
"""合并查询节点:读上游的查询快照,在库外按键合并,结果存成和数据源查询同形的查询快照。
|
||||
|
||||
**读的是工件库里那份完整结果。** 上游 tool 节点交回的 JSON 里带着查询快照的工件 id,这里按 id 取回快照(取回时复验
|
||||
哈希),不读交给模型的那份文字:Agent 的消息历史里结果可能被截短,拿截短的文字合并会悄悄少行。上游是另一个合并
|
||||
节点时同理,取它的结果快照。
|
||||
|
||||
**结果和数据源查询同形。** 快照有 columns、rows、truncated、sql、source(记为「合并查询」),台账里记一条 kind 为
|
||||
query 的条目:报告照样按先后编成 Qn,[[v:Qn.rK.列]]、[[table:Qn …]] 和口径卡的 cell(nodes.<本节点>, 行, '列') 都按
|
||||
原来的规则取值、核对。快照另记 inputs(别名、节点、各自的快照)和合并 SQL,证据面板据此展示合并的来源;逐格来历
|
||||
记在 lineage 里(见 engine/merge_query.py),下钻据此指回输入的那一格。
|
||||
|
||||
**重放结果一致。** 快照里没有耗时这类每次不同的字段:节点重放、继续运行时同样的输入得到同一个工件 id、同一条
|
||||
台账,报告里已经写下的出处不会因此变成另一份快照。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.core.events import EventType
|
||||
from app.data.guard import QueryLimits
|
||||
from app.engine.context import NodeContext, NodeError
|
||||
from app.engine.evidence import ledger_enabled, next_exec
|
||||
from app.engine.merge_query import MERGE_SOURCE, MergeError, MergeInput, execute
|
||||
from app.engine.state import GraphState
|
||||
|
||||
#: 合并警告进运行事件时的 code(log 事件)。运行详情、运行概况按它认出这是合并查询的警告
|
||||
LOG_CODES = {"key_type_mismatch": "merge_key_type", "rows_grew": "merge_rows_grew",
|
||||
"result_truncated": "merge_truncated"}
|
||||
#: merge.end 事件里带几行预览:运行详情只是让人看一眼合并出了什么,完整结果在查询快照里
|
||||
PREVIEW_ROWS = 5
|
||||
#: 上游查询没查成时,tool 节点交回的原话的开头(engine/nodes/tools.py 的 QUERY_FAILED 同一组)
|
||||
_FAILED_PREFIXES = ("查询失败", "SQL 被拒绝:")
|
||||
|
||||
|
||||
def _input_payload(value: Any) -> dict[str, Any] | None:
|
||||
"""上游节点的产出 → 带工件 id 的查询结果(tool 节点交回的是 JSON 文本,合并节点交回的是对象)。"""
|
||||
data = value
|
||||
if isinstance(data, str):
|
||||
try:
|
||||
data = json.loads(data)
|
||||
except ValueError:
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
async def _load_input(state: GraphState, ctx: NodeContext, alias: str, node_id: Any) -> MergeInput:
|
||||
nodes = ctx.run.spec.node_map()
|
||||
target = nodes.get(node_id) if isinstance(node_id, str) else None
|
||||
if target is None:
|
||||
raise NodeError(ctx.node.id, f"输入「{alias}」选择的节点「{node_id}」不存在,请在节点中重新选择")
|
||||
who = f"输入「{alias}」({target.title})"
|
||||
value = (state.get("nodes") or {}).get(target.id)
|
||||
if value is None:
|
||||
raise NodeError(ctx.node.id, f"{who}还没有运行结果:请确认它连在合并查询之前")
|
||||
if isinstance(value, dict) and value.get("skipped"):
|
||||
raise NodeError(ctx.node.id, f"{who}满足跳过条件被跳过了,没有查询结果可以合并")
|
||||
if isinstance(value, dict) and value.get("failed"):
|
||||
raise NodeError(ctx.node.id, f"{who}执行失败,没有查询结果可以合并:{value.get('error') or '原因未知'}")
|
||||
if isinstance(value, str) and value.startswith(_FAILED_PREFIXES):
|
||||
raise NodeError(ctx.node.id, f"{who}的查询没有成功,无法合并:{value[:300]}")
|
||||
payload = _input_payload(value)
|
||||
artifact = payload.get("artifact") if payload else None
|
||||
if not isinstance(artifact, str) or not artifact:
|
||||
raise NodeError(ctx.node.id, f"{who}的结果不是存成快照的查询结果。合并查询只能合并数据库查询(调用数据库查询工具的"
|
||||
"「调用工具」节点)或另一个「合并查询」节点的结果")
|
||||
try:
|
||||
snapshot = await asyncio.to_thread(artifact_store.load, artifact)
|
||||
except ValueError:
|
||||
raise NodeError(ctx.node.id, f"{who}的查询快照与哈希不一致,疑似被修改,不能合并") from None
|
||||
if snapshot is None:
|
||||
raise NodeError(ctx.node.id, f"{who}的查询快照已不存在,无法合并。请重新运行它")
|
||||
return MergeInput(alias=alias, node_id=target.id, artifact=artifact, snapshot=snapshot, label=target.title)
|
||||
|
||||
|
||||
async def run_merge(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
inputs = ctx.config.get("inputs")
|
||||
if not isinstance(inputs, dict) or not inputs:
|
||||
raise NodeError(ctx.node.id, "「合并查询」还没有配置输入:至少选择一个上游的查询节点,并给它起一个别名")
|
||||
# SQL 里允许 {{ }}(例如按运行输入筛日期),和 tool 节点的 SQL 一样先渲染;快照里存的是渲染之后、守卫规范过的那条
|
||||
sql = ctx.render_str(str(ctx.config.get("sql") or ""), state)
|
||||
loaded = [await _load_input(state, ctx, str(alias), node_id) for alias, node_id in inputs.items()]
|
||||
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
# 内存 SQLite 是同步的:放进线程,大一点的合并不卡住事件循环(别的节点的事件照常流出去)
|
||||
outcome = await asyncio.to_thread(execute, loaded, sql, limits=QueryLimits())
|
||||
except MergeError as e:
|
||||
raise NodeError(ctx.node.id, str(e)) from None
|
||||
elapsed = int((time.perf_counter() - started) * 1000)
|
||||
|
||||
snapshot = outcome.snapshot()
|
||||
try:
|
||||
artifact = await artifact_store.put_json(snapshot, kind="query_snapshot", run_id=ctx.run.run_id,
|
||||
node_id=ctx.node.id)
|
||||
except Exception as e: # noqa: BLE001 - 存不下就没有出处可言,不能带着一份引用不了的结果往下走
|
||||
raise NodeError(ctx.node.id, "合并结果没能存成查询快照(工件库写入失败),下游无法引用它。"
|
||||
"请稍后点「继续运行」重试", retryable=True) from e
|
||||
|
||||
labels = {inp.alias: inp.label for inp in loaded}
|
||||
inputs_view = [{**{k: v for k, v in item.items() if k != "columns"}, "label": labels.get(item["alias"]) or None}
|
||||
for item in outcome.inputs]
|
||||
ctx.emit(EventType.MERGE_END, inputs=inputs_view, sql=outcome.sql, rows=len(outcome.rows),
|
||||
columns=outcome.columns, truncated=outcome.truncated, warnings=outcome.warnings,
|
||||
preview_rows=outcome.rows[:PREVIEW_ROWS], duration_ms=elapsed, query_artifact=artifact,
|
||||
lineage=outcome.lineage is not None)
|
||||
for warning in outcome.warnings:
|
||||
ctx.emit(EventType.LOG, level="warn", message=warning["message"],
|
||||
code=LOG_CODES.get(warning["code"], "merge_warning"))
|
||||
|
||||
output: dict[str, Any] = {
|
||||
"artifact": artifact, "columns": outcome.columns, "rows": outcome.rows, "row_count": len(outcome.rows),
|
||||
"truncated": outcome.truncated, "sql": outcome.sql, "source": MERGE_SOURCE, "inputs": inputs_view,
|
||||
"warnings": outcome.warnings,
|
||||
}
|
||||
if outcome.column_types:
|
||||
output["column_types"] = outcome.column_types
|
||||
updates: dict[str, Any] = {"nodes": {ctx.node.id: output}}
|
||||
if ledger_enabled(ctx.run):
|
||||
# 证据台账:和数据源查询同形的 query 条目,报告据此编 Qn;没有 via(外面不包工具快照)
|
||||
updates["evidence"] = [{
|
||||
"kind": "query", "node_id": ctx.node.id, "exec": next_exec(state, ctx.node.id), "artifact": artifact,
|
||||
"source": MERGE_SOURCE, "columns": outcome.columns, "rows": len(outcome.rows),
|
||||
"truncated": outcome.truncated,
|
||||
}]
|
||||
var_name = str(ctx.config.get("assign_to") or "").strip()
|
||||
if var_name:
|
||||
updates["vars"] = {var_name: output}
|
||||
return updates
|
||||
@@ -330,7 +330,7 @@ _REF_HEAD = re.compile(r"^(vars|nodes|input)\.([A-Za-z_一-鿿][\w\-一-鿿]*)(.
|
||||
_VIA = {"input": "input", "code": "code", "agent": "agent", "llm": "llm", "tool": "tool",
|
||||
"transform": "transform", "metrics": "metric", "retrieve": "retrieval", "loop": "loop",
|
||||
"human": "human", "validate": "validate", "supervisor": "team", "subgraph": "subgraph",
|
||||
"memory": "memory"}
|
||||
"memory": "memory", "merge": "merge"}
|
||||
|
||||
|
||||
def _input_of(path: str, value: Any, state: GraphState, ctx: NodeContext,
|
||||
|
||||
@@ -23,6 +23,7 @@ class NodeType(StrEnum):
|
||||
|
||||
TOOL = "tool" # 直接调用一个工具
|
||||
CODE = "code" # 沙箱里跑代码
|
||||
MERGE = "merge" # 合并查询:几次查询的完整结果在库外按键合并(engine/merge_query.py)
|
||||
|
||||
BRANCH = "branch" # 条件分支
|
||||
LOOP = "loop" # 循环 / 迭代
|
||||
|
||||
@@ -53,7 +53,7 @@ _LLM_ONLY = ("prompt", "system", "skills", "use_history", "output_schema")
|
||||
_MODEL_KEYS = ("provider", "model")
|
||||
#: 报告的指令里照搬了这几类节点的文字,就要提醒:报告撰写自己会收集它们的证据
|
||||
_EVIDENCE_TYPES = (NodeType.METRICS, NodeType.TOOL, NodeType.AGENT, NodeType.RETRIEVE, NodeType.CODE,
|
||||
NodeType.SUBGRAPH)
|
||||
NodeType.SUBGRAPH, NodeType.MERGE)
|
||||
#: 写整段文字的节点:出具契约的 narrative 拼了其中几个的文字,就说不准该换哪一个
|
||||
_WRITER_TYPES = (NodeType.LLM, NodeType.AGENT)
|
||||
_QUESTION = re.compile(r"(?<![\w.])input\s*\.\s*question(?![\w-])")
|
||||
@@ -93,7 +93,9 @@ class _Step:
|
||||
|
||||
def _queries(node: GraphNode) -> bool:
|
||||
"""这个节点查库:报告撰写能把它查到的结果编成 Q1、Q2… 引用。只认数据源的查询工具,别的工具的
|
||||
返回进不了目录。"""
|
||||
返回进不了目录。合并查询的结果和数据源查询同形,同样编号。"""
|
||||
if node.type == NodeType.MERGE:
|
||||
return True
|
||||
if node.type == NodeType.TOOL:
|
||||
return str(node.config.get("tool") or "").startswith("db_query__")
|
||||
if node.type == NodeType.AGENT:
|
||||
|
||||
@@ -340,6 +340,15 @@ def analyze(spec: GraphSpec) -> VariableReport:
|
||||
node_id=node.id, node_label=label, field=field, expr=m.group(0),
|
||||
))
|
||||
|
||||
# 合并查询的输入按节点 id 选,不写模板:不认这一条的话,变量表里看不出上游那条查询被谁用了。
|
||||
# 选错的节点(不存在、不在上游、不是查询)由 validate_graph 报,这里只记引用
|
||||
if node.type == "merge" and isinstance(cfg.get("inputs"), dict):
|
||||
for alias, source in cfg["inputs"].items():
|
||||
target = index.get(f"nodes.{source}") if isinstance(source, str) else None
|
||||
if target is not None:
|
||||
target.refs.append(VarRef(node_id=node.id, node_label=label, field=f"inputs.{alias}",
|
||||
expr=f"nodes.{source}"))
|
||||
|
||||
for field, text in _iter_strings(cfg):
|
||||
for m in _TEMPLATE_RE.finditer(text):
|
||||
expr = m.group(1).strip()
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
-- 会员库(虚构):会员到店记录,和门店库是两个独立的库。门店编号同样是文本,但列名不同(store_code)。
|
||||
-- 合并查询的端到端测试按「日期 + 门店」聚合到店人数,再和门店库的订单按键合并。数据只为测试编造。
|
||||
CREATE TABLE visits (
|
||||
visit_id INTEGER PRIMARY KEY,
|
||||
member_id INTEGER NOT NULL,
|
||||
store_code TEXT NOT NULL,
|
||||
visit_date TEXT NOT NULL
|
||||
);
|
||||
INSERT INTO visits VALUES
|
||||
(1, 101, 'S01', '2026-05-01'),
|
||||
(2, 102, 'S01', '2026-05-01'),
|
||||
(3, 103, 'S01', '2026-05-01'),
|
||||
(4, 104, 'S01', '2026-05-01'),
|
||||
(5, 105, 'S01', '2026-05-01'),
|
||||
(6, 106, 'S01', '2026-05-01'),
|
||||
(7, 201, 'S02', '2026-05-01'),
|
||||
(8, 202, 'S02', '2026-05-01'),
|
||||
(9, 203, 'S02', '2026-05-01'),
|
||||
(10, 204, 'S02', '2026-05-01'),
|
||||
(11, 101, 'S01', '2026-05-02'),
|
||||
(12, 107, 'S01', '2026-05-02'),
|
||||
(13, 108, 'S01', '2026-05-02'),
|
||||
(14, 109, 'S01', '2026-05-02'),
|
||||
(15, 110, 'S01', '2026-05-02'),
|
||||
(16, 201, 'S02', '2026-05-02'),
|
||||
(17, 205, 'S02', '2026-05-02'),
|
||||
(18, 206, 'S02', '2026-05-02'),
|
||||
(19, 207, 'S02', '2026-05-02');
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
-- 门店库(虚构):门店和销售订单。合并查询的端到端测试按「日期 + 门店」聚合订单数和销售额。
|
||||
-- 门店编号是文本('S01'),金额是实数。数据只为测试编造,和任何真实门店无关。
|
||||
CREATE TABLE stores (
|
||||
store_id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE orders (
|
||||
order_id INTEGER PRIMARY KEY,
|
||||
store_id TEXT NOT NULL REFERENCES stores (store_id),
|
||||
order_date TEXT NOT NULL,
|
||||
amount REAL NOT NULL
|
||||
);
|
||||
INSERT INTO stores VALUES ('S01', '湖滨店'), ('S02', '东站店');
|
||||
INSERT INTO orders VALUES
|
||||
(1, 'S01', '2026-05-01', 120.0),
|
||||
(2, 'S01', '2026-05-01', 80.0),
|
||||
(3, 'S01', '2026-05-01', 100.0),
|
||||
(4, 'S02', '2026-05-01', 60.0),
|
||||
(5, 'S02', '2026-05-01', 90.0),
|
||||
(6, 'S01', '2026-05-02', 200.0),
|
||||
(7, 'S01', '2026-05-02', 50.0),
|
||||
(8, 'S02', '2026-05-02', 75.0);
|
||||
@@ -0,0 +1,385 @@
|
||||
"""合并查询的库外合并(engine/merge_query.py):几次查询的完整快照落进内存 SQLite,再按一条 SELECT / WITH 合并。
|
||||
|
||||
纯函数,不连库、不读工件:快照在测试里现造。端到端(真运行、证据面板、推断来源)见 test_merge_node.py。
|
||||
|
||||
场景是虚构的门店经营:门店库按「日期 + 门店」聚合的销售,会员库按同样粒度聚合的到店人数。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.data.guard import QueryLimits
|
||||
from app.engine.merge_query import (
|
||||
MERGE_SOURCE,
|
||||
MergeError,
|
||||
MergeInput,
|
||||
alias_problem,
|
||||
execute,
|
||||
sql_problem,
|
||||
trace_cell,
|
||||
)
|
||||
from app.engine.expressions import same_value
|
||||
|
||||
SALES_COLS = ["日期", "门店", "订单数", "销售额"]
|
||||
SALES = [
|
||||
["2026-05-01", "S01", 40, "3200.50"],
|
||||
["2026-05-01", "S02", 25, "1875.00"],
|
||||
["2026-05-02", "S01", 38, "3010.00"],
|
||||
["2026-05-02", "S02", 30, "2400.00"],
|
||||
]
|
||||
VISITS_COLS = ["日期", "门店", "到店人数"]
|
||||
VISITS = [
|
||||
["2026-05-01", "S01", 160],
|
||||
["2026-05-01", "S02", 90],
|
||||
["2026-05-02", "S02", 100],
|
||||
["2026-05-02", "S01", 150],
|
||||
]
|
||||
JOIN_SQL = ("SELECT s.日期, s.门店, s.订单数, s.销售额, v.到店人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 "
|
||||
"ORDER BY s.日期, s.门店")
|
||||
|
||||
|
||||
def snap(columns, rows, *, types=None, truncated=False, source="stores", **extra):
|
||||
return {"columns": list(columns), "rows": [list(r) for r in rows], "row_count": len(rows), "truncated": truncated,
|
||||
"sql": "SELECT …", "source": source, **({"column_types": types} if types is not None else {}), **extra}
|
||||
|
||||
|
||||
def sales(**kw):
|
||||
return MergeInput(alias="s", node_id="q_sales", artifact="a" * 64,
|
||||
snapshot=snap(SALES_COLS, SALES, types={"日期": "text", "门店": "text", "订单数": "number",
|
||||
"销售额": "number"}, **kw))
|
||||
|
||||
|
||||
def visits(rows=VISITS, *, types=None, **kw):
|
||||
return MergeInput(alias="v", node_id="q_visits", artifact="b" * 64,
|
||||
snapshot=snap(VISITS_COLS, rows, types=types or {"日期": "text", "门店": "text", "到店人数": "number"},
|
||||
source="members", **kw))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 合并本身
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_join_on_date_and_store():
|
||||
out = execute([sales(), visits()], JOIN_SQL)
|
||||
assert out.columns == ["日期", "门店", "订单数", "销售额", "到店人数"]
|
||||
assert out.rows == [
|
||||
["2026-05-01", "S01", 40, 3200.5, 160],
|
||||
["2026-05-01", "S02", 25, 1875, 90],
|
||||
["2026-05-02", "S01", 38, 3010, 150],
|
||||
["2026-05-02", "S02", 30, 2400, 100],
|
||||
]
|
||||
# 数值列按数存:数据源把 DECIMAL 落成的文本("3200.50")合并之后能直接参与运算
|
||||
assert out.column_types == {"日期": "text", "门店": "text", "订单数": "number", "销售额": "number",
|
||||
"到店人数": "number"}
|
||||
assert out.truncated is False and out.warnings == []
|
||||
assert out.sql == JOIN_SQL
|
||||
|
||||
|
||||
def test_with_is_allowed_and_text_keeps_leading_zeros():
|
||||
a = MergeInput(alias="a", node_id="n1", artifact="c" * 64,
|
||||
snapshot=snap(["会员号", "积分"], [["007", 10], ["070", 20]], types={"会员号": "text", "积分": "number"}))
|
||||
out = execute([a], "WITH t AS (SELECT 会员号, 积分 FROM a) SELECT 会员号, 积分 * 2 AS 双倍 FROM t ORDER BY 会员号")
|
||||
assert out.rows == [["007", 20], ["070", 40]]
|
||||
|
||||
|
||||
def test_column_names_are_checked_not_treated_as_strings():
|
||||
"""内存库同样关掉「双引号当字符串」:写错的列名要报错,不能静默得出一列常量。"""
|
||||
with pytest.raises(MergeError, match="no such column"):
|
||||
execute([sales()], 'SELECT "销售颔" FROM s')
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", [
|
||||
"DELETE FROM s",
|
||||
"UPDATE s SET 订单数 = 0",
|
||||
"INSERT INTO s VALUES (1, 2, 3, 4)",
|
||||
"DROP TABLE s",
|
||||
"PRAGMA table_info(s)",
|
||||
"ATTACH DATABASE '/tmp/x.db' AS x",
|
||||
"WITH k AS (SELECT 1) DELETE FROM s",
|
||||
"EXPLAIN SELECT * FROM s",
|
||||
])
|
||||
def test_only_select_and_with(sql):
|
||||
assert sql_problem(sql) is not None
|
||||
with pytest.raises(MergeError):
|
||||
execute([sales()], sql)
|
||||
|
||||
|
||||
def test_one_statement_only():
|
||||
problem = sql_problem("SELECT * FROM s; SELECT * FROM v")
|
||||
assert problem is not None and "一条" in problem
|
||||
with pytest.raises(MergeError):
|
||||
execute([sales(), visits()], "SELECT * FROM s; SELECT * FROM v")
|
||||
|
||||
|
||||
def test_empty_sql():
|
||||
assert "合并 SQL" in sql_problem(" ")
|
||||
|
||||
|
||||
def test_static_check_tolerates_templates():
|
||||
assert sql_problem("SELECT * FROM s WHERE 日期 >= {{ input.start | json }}") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alias, ok", [
|
||||
("s", True), ("sales_2026", True), ("_t", True),
|
||||
("", False), ("1s", False), ("a-b", False), ("a b", False), ("订单", False),
|
||||
("select", False), ("ORDER", False), ("join", False), ("sqlite_stat", False), ("x" * 40, False),
|
||||
])
|
||||
def test_alias_must_be_a_plain_table_name(alias, ok):
|
||||
assert (alias_problem(alias) is None) is ok
|
||||
|
||||
|
||||
def test_aliases_differing_only_in_case_collide():
|
||||
a = MergeInput(alias="t", node_id="n1", artifact="c" * 64, snapshot=snap(["x"], [[1]]))
|
||||
b = MergeInput(alias="T", node_id="n2", artifact="d" * 64, snapshot=snap(["x"], [[2]]))
|
||||
with pytest.raises(MergeError, match="别名"):
|
||||
execute([a, b], "SELECT * FROM t")
|
||||
|
||||
|
||||
def test_duplicate_column_names_in_an_input_are_refused_with_a_fix():
|
||||
a = MergeInput(alias="t", node_id="n1", artifact="c" * 64, snapshot=snap(["金额", "金额"], [[1, 2]]))
|
||||
with pytest.raises(MergeError, match="重名"):
|
||||
execute([a], "SELECT * FROM t")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 三道防护
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_truncated_input_is_refused():
|
||||
"""被截断的输入只有前若干行:拿它按键合并,缺的行不会报错,只会悄悄少算。"""
|
||||
with pytest.raises(MergeError) as e:
|
||||
execute([sales(), visits(truncated=True)], JOIN_SQL)
|
||||
message = str(e.value)
|
||||
assert "「v」" in message and "截断" in message
|
||||
assert "聚合" in message and "缩小范围" in message
|
||||
|
||||
|
||||
def test_key_type_mismatch_is_warned():
|
||||
"""门店编号一边是文本 '001',一边是数 1:SQLite 比较时会隐式转换,'001'、'01' 都等于 1。"""
|
||||
stores = MergeInput(alias="s", node_id="q_sales", artifact="a" * 64,
|
||||
snapshot=snap(["门店", "销售额"], [["001", 100], ["002", 200]],
|
||||
types={"门店": "text", "销售额": "number"}))
|
||||
traffic = MergeInput(alias="v", node_id="q_visits", artifact="b" * 64,
|
||||
snapshot=snap(["门店", "到店人数"], [[1, 10], [2, 20]], types={"门店": "number", "到店人数": "number"},
|
||||
source="members"))
|
||||
out = execute([stores, traffic], "SELECT s.门店, s.销售额, v.到店人数 FROM s JOIN v ON s.门店 = v.门店")
|
||||
[warning] = out.warnings
|
||||
assert warning["code"] == "key_type_mismatch"
|
||||
assert "s.门店" in warning["message"] and "v.门店" in warning["message"]
|
||||
assert "'001'" in warning["message"] and "CAST" in warning["message"]
|
||||
|
||||
|
||||
def test_key_type_mismatch_is_found_through_using():
|
||||
stores = MergeInput(alias="s", node_id="n1", artifact="a" * 64,
|
||||
snapshot=snap(["门店", "销售额"], [["1", 100]], types={"门店": "text", "销售额": "number"}))
|
||||
traffic = MergeInput(alias="v", node_id="n2", artifact="b" * 64,
|
||||
snapshot=snap(["门店", "到店人数"], [[1, 10]], types={"门店": "number", "到店人数": "number"}))
|
||||
out = execute([stores, traffic], "SELECT 门店, 销售额, 到店人数 FROM s JOIN v USING (门店)")
|
||||
assert [w["code"] for w in out.warnings] == ["key_type_mismatch"]
|
||||
|
||||
|
||||
def test_matching_key_types_raise_no_warning():
|
||||
out = execute([sales(), visits()], JOIN_SQL)
|
||||
assert out.warnings == []
|
||||
|
||||
|
||||
def test_more_rows_than_the_largest_input_is_warned():
|
||||
"""两边都有一天同一门店出现两行(没聚合到同一粒度):多对多匹配,合并后行数被放大。"""
|
||||
stores = sales()
|
||||
stores.snapshot["rows"].append(["2026-05-01", "S01", 2, "100.00"])
|
||||
doubled = [*VISITS, ["2026-05-01", "S01", 5]]
|
||||
out = execute([stores, visits(doubled)], JOIN_SQL)
|
||||
assert len(out.rows) == 7
|
||||
grew = [w for w in out.warnings if w["code"] == "rows_grew"]
|
||||
assert len(grew) == 1
|
||||
message = grew[0]["message"]
|
||||
assert "7 行" in message and "5 行" in message and "不唯一" in message
|
||||
# 合并键认得出来时,说出是哪个输入、哪组键重复
|
||||
assert "「v」" in message and "2026-05-01" in message and "S01" in message
|
||||
|
||||
|
||||
def test_one_to_many_is_not_warned_as_growth():
|
||||
"""一边的键唯一时,结果行数不会超过另一边:不报。"""
|
||||
doubled = [*VISITS, ["2026-05-01", "S01", 5]]
|
||||
out = execute([sales(), visits(doubled)], JOIN_SQL)
|
||||
assert len(out.rows) == 5 and out.warnings == []
|
||||
|
||||
|
||||
def test_result_over_the_row_limit_is_truncated_and_warned():
|
||||
out = execute([sales(), visits()], "SELECT * FROM s CROSS JOIN v", limits=QueryLimits(max_rows=5))
|
||||
assert out.truncated is True and len(out.rows) == 5
|
||||
assert "result_truncated" in [w["code"] for w in out.warnings]
|
||||
|
||||
|
||||
def test_byte_limit_matches_the_query_layer():
|
||||
out = execute([sales(), visits()], "SELECT * FROM s CROSS JOIN v", limits=QueryLimits(max_bytes=100))
|
||||
assert out.truncated is True and 0 < len(out.rows) < 16
|
||||
|
||||
|
||||
def test_runaway_query_is_stopped():
|
||||
endless = "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c) SELECT count(*) FROM c"
|
||||
with pytest.raises(MergeError, match="秒"):
|
||||
execute([sales()], endless, limits=QueryLimits(timeout_seconds=0.2))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 逐格来历:宁可不下钻,也不下钻错
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _assert_traces_are_true(inputs, out):
|
||||
"""凡是给出来历的格,都和它指向的那个输入的那一格是同一个值。"""
|
||||
by_alias = {i.alias: i for i in inputs}
|
||||
found = 0
|
||||
for r, row in enumerate(out.rows):
|
||||
for column in out.columns:
|
||||
hit = trace_cell(out.snapshot(), r, column)
|
||||
if hit is None:
|
||||
continue
|
||||
found += 1
|
||||
source = by_alias[hit.alias].snapshot
|
||||
value = source["rows"][hit.row][source["columns"].index(hit.column)]
|
||||
assert same_value(row[out.columns.index(column)], value), (r, column, hit)
|
||||
return found
|
||||
|
||||
|
||||
def test_direct_columns_of_a_join_trace_to_the_input_rows():
|
||||
inputs = [sales(), visits()]
|
||||
out = execute(inputs, JOIN_SQL)
|
||||
assert out.lineage is not None
|
||||
# 2026-05-02 / S01 在会员库里是第 3 行(行号从 0 数),在门店库里是第 2 行
|
||||
hit = trace_cell(out.snapshot(), 2, "到店人数")
|
||||
assert (hit.alias, hit.row, hit.column) == ("v", 3, "到店人数")
|
||||
hit = trace_cell(out.snapshot(), 2, "销售额")
|
||||
assert (hit.alias, hit.row, hit.column) == ("s", 2, "销售额")
|
||||
assert _assert_traces_are_true(inputs, out) == 20
|
||||
|
||||
|
||||
def test_renamed_columns_still_trace():
|
||||
inputs = [sales(), visits()]
|
||||
out = execute(inputs, "SELECT s.门店 AS 店, v.到店人数 人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 "
|
||||
"WHERE s.日期 = '2026-05-02' ORDER BY 1")
|
||||
hit = trace_cell(out.snapshot(), 1, "人数")
|
||||
assert (hit.alias, hit.row, hit.column) == ("v", 2, "到店人数")
|
||||
assert _assert_traces_are_true(inputs, out) == 4
|
||||
|
||||
|
||||
def test_computed_columns_have_no_trace():
|
||||
inputs = [sales(), visits()]
|
||||
out = execute(inputs, "SELECT s.门店, s.订单数 * 1.0 / v.到店人数 AS 转化率 FROM s JOIN v "
|
||||
"ON s.日期 = v.日期 AND s.门店 = v.门店 ORDER BY s.日期, s.门店")
|
||||
assert trace_cell(out.snapshot(), 0, "转化率") is None
|
||||
assert trace_cell(out.snapshot(), 0, "门店").alias == "s"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", [
|
||||
"SELECT 日期, sum(订单数) AS 订单数 FROM s GROUP BY 日期",
|
||||
"SELECT DISTINCT 门店 FROM s",
|
||||
"SELECT 门店 FROM s UNION SELECT 门店 FROM v",
|
||||
"SELECT 门店, (SELECT max(到店人数) FROM v) AS 峰值 FROM s",
|
||||
"SELECT max(订单数) AS 订单数 FROM s",
|
||||
"WITH t AS (SELECT * FROM s) SELECT 订单数 FROM t",
|
||||
"SELECT 门店, 订单数, row_number() OVER (ORDER BY 订单数) AS 名次 FROM s",
|
||||
"SELECT * FROM s NATURAL JOIN v",
|
||||
])
|
||||
def test_shapes_outside_the_grammar_have_no_trace_at_all(sql):
|
||||
out = execute([sales(), visits()], sql)
|
||||
assert out.lineage is None
|
||||
assert all(trace_cell(out.snapshot(), r, c) is None for r in range(len(out.rows)) for c in out.columns)
|
||||
|
||||
|
||||
def test_left_join_misses_have_no_trace_on_the_missing_side():
|
||||
short = [["2026-05-01", "S01", 160]]
|
||||
inputs = [sales(), visits(short)]
|
||||
out = execute(inputs, "SELECT s.日期, s.门店, v.到店人数 FROM s LEFT JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 "
|
||||
"ORDER BY s.日期, s.门店")
|
||||
assert out.rows[1][2] is None
|
||||
assert trace_cell(out.snapshot(), 1, "到店人数") is None
|
||||
assert trace_cell(out.snapshot(), 1, "门店").alias == "s"
|
||||
assert trace_cell(out.snapshot(), 0, "到店人数").row == 0
|
||||
_assert_traces_are_true(inputs, out)
|
||||
|
||||
|
||||
def test_star_expands_in_from_order():
|
||||
inputs = [sales(), visits()]
|
||||
out = execute(inputs, "SELECT * FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 ORDER BY s.日期, s.门店")
|
||||
assert out.columns == [*SALES_COLS, *VISITS_COLS]
|
||||
# 重名的列按第一次出现引用:「日期」指的是门店库那一列
|
||||
assert trace_cell(out.snapshot(), 0, "日期").alias == "s"
|
||||
assert trace_cell(out.snapshot(), 0, "到店人数").alias == "v"
|
||||
_assert_traces_are_true(inputs, out)
|
||||
|
||||
|
||||
def test_using_columns_written_bare_have_no_trace():
|
||||
"""USING 合并掉的列不写表名时,SQLite 取的是哪一边说不准(RIGHT / FULL JOIN 时是两边合起来的值)。"""
|
||||
out = execute([sales(), visits()], "SELECT 日期, 门店, 订单数, 到店人数 FROM s JOIN v USING (日期, 门店) ORDER BY 1, 2")
|
||||
assert trace_cell(out.snapshot(), 0, "日期") is None
|
||||
assert trace_cell(out.snapshot(), 0, "订单数").alias == "s"
|
||||
|
||||
|
||||
def test_identical_output_rows_are_not_traced():
|
||||
"""两行结果一模一样时,说不清哪一行来自哪一行输入:不给。"""
|
||||
twins = [["2026-05-01", "S01", 7], ["2026-05-01", "S01", 7], ["2026-05-02", "S02", 9]]
|
||||
inputs = [MergeInput(alias="v", node_id="n", artifact="b" * 64, snapshot=snap(VISITS_COLS, twins))]
|
||||
out = execute(inputs, "SELECT 日期, 门店, 到店人数 FROM v")
|
||||
assert trace_cell(out.snapshot(), 0, "到店人数") is None and trace_cell(out.snapshot(), 1, "到店人数") is None
|
||||
assert trace_cell(out.snapshot(), 2, "到店人数").row == 2
|
||||
|
||||
|
||||
def test_self_join_traces_each_side_separately():
|
||||
inputs = [visits()]
|
||||
out = execute(inputs, "SELECT a.门店, a.到店人数 AS 首日, b.到店人数 AS 次日 FROM v AS a JOIN v AS b "
|
||||
"ON a.门店 = b.门店 AND a.日期 = '2026-05-01' AND b.日期 = '2026-05-02' ORDER BY a.门店")
|
||||
assert out.rows == [["S01", 160, 150], ["S02", 90, 100]]
|
||||
first, second = trace_cell(out.snapshot(), 0, "首日"), trace_cell(out.snapshot(), 0, "次日")
|
||||
assert (first.alias, first.row) == ("v", 0) and (second.alias, second.row) == ("v", 3)
|
||||
_assert_traces_are_true(inputs, out)
|
||||
|
||||
|
||||
def test_a_rowid_column_in_the_input_does_not_confuse_the_trace():
|
||||
rows = [[9, "S01", 1], [8, "S02", 2]]
|
||||
inputs = [MergeInput(alias="t", node_id="n", artifact="b" * 64, snapshot=snap(["rowid", "门店", "值"], rows))]
|
||||
out = execute(inputs, "SELECT 门店, 值 FROM t ORDER BY 门店 DESC")
|
||||
assert out.rows == [["S02", 2], ["S01", 1]]
|
||||
assert trace_cell(out.snapshot(), 0, "值").row == 1
|
||||
_assert_traces_are_true(inputs, out)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 快照形状
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_snapshot_has_the_query_shape_plus_inputs():
|
||||
out = execute([sales(), visits()], JOIN_SQL)
|
||||
snapshot = out.snapshot()
|
||||
for key in ("columns", "rows", "row_count", "truncated", "sql", "column_types"):
|
||||
assert key in snapshot
|
||||
assert snapshot["source"] == MERGE_SOURCE
|
||||
assert snapshot["inputs"] == [
|
||||
{"alias": "s", "node_id": "q_sales", "artifact": "a" * 64, "rows": 4, "columns": SALES_COLS, "source": "stores"},
|
||||
{"alias": "v", "node_id": "q_visits", "artifact": "b" * 64, "rows": 4, "columns": VISITS_COLS,
|
||||
"source": "members"},
|
||||
]
|
||||
assert snapshot["sources"] == ["stores", "members"]
|
||||
# 内容确定:同样的输入、同样的 SQL,快照逐字相同(重放、继续运行得到同一个工件)
|
||||
assert execute([sales(), visits()], JOIN_SQL).snapshot() == snapshot
|
||||
|
||||
|
||||
def test_masked_input_columns_stay_masked_even_when_renamed():
|
||||
members = MergeInput(alias="v", node_id="q_visits", artifact="b" * 64,
|
||||
snapshot=snap(["门店", "手机号"], [["S01", "138****"]], source="members",
|
||||
mask_columns=["手机号"]))
|
||||
out = execute([sales(), members], "SELECT s.门店, v.手机号 AS 联系方式 FROM s JOIN v ON s.门店 = v.门店")
|
||||
assert set(out.snapshot()["mask_columns"]) == {"手机号", "联系方式"}
|
||||
|
||||
|
||||
def test_nested_merges_keep_the_underlying_sources():
|
||||
first = execute([sales(), visits()], JOIN_SQL)
|
||||
nested = MergeInput(alias="m", node_id="merge1", artifact="e" * 64, snapshot=first.snapshot())
|
||||
out = execute([nested], "SELECT 日期, 门店, 到店人数 FROM m WHERE 门店 = 'S02'")
|
||||
assert out.snapshot()["sources"] == ["stores", "members"]
|
||||
hit = trace_cell(out.snapshot(), 0, "到店人数")
|
||||
assert (hit.alias, hit.row) == ("m", 1)
|
||||
@@ -0,0 +1,456 @@
|
||||
"""合并查询节点(merge):画图时的校验、执行、证据链和下钻、助手说明。
|
||||
|
||||
端到端用两个独立的 SQLite 夹具(tests/fixtures/merge/):门店库的销售订单、会员库的到店记录。两条查询各自在自己的
|
||||
库里聚合到「日期 + 门店」,合并查询在库外按键合并,口径卡用 cell(nodes.merge, …) 算转化率和客单价,报告引用合并
|
||||
结果的单元格和口径卡的指标。断言报告里的数能追到两边的原始查询,下钻不指错格子。
|
||||
|
||||
不调用任何模型:报告的写作者换成脚本(monkeypatch MockChatModel._decide)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from langchain_core.messages import AIMessage
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.core.config import settings
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource, Run, RunEvent
|
||||
from app.engine.evidence import iter_segments
|
||||
from app.engine.merge_query import MERGE_SOURCE
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import GraphSpec, NodeType, validate_graph
|
||||
from app.main import app
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures" / "merge"
|
||||
|
||||
SALES_SQL = ("SELECT order_date AS 日期, store_id AS 门店, COUNT(*) AS 订单数, SUM(amount) AS 销售额 FROM orders "
|
||||
"GROUP BY order_date, store_id ORDER BY order_date, store_id")
|
||||
# 故意和门店库排成不同的顺序:合并结果的第 0 行在会员库的结果里是第 2 行
|
||||
VISITS_SQL = ("SELECT visit_date AS 日期, store_code AS 门店, COUNT(*) AS 到店人数 FROM visits "
|
||||
"GROUP BY visit_date, store_code ORDER BY visit_date DESC, store_code")
|
||||
MERGE_SQL = ("SELECT s.日期, s.门店, s.订单数, s.销售额, v.到店人数 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 "
|
||||
"ORDER BY s.日期, s.门店")
|
||||
TEXT = ("5 月 1 日湖滨店到店 [[v:Q3.r0.到店人数]] 人,转化率 [[m:conversion]],客单价 [[m:aov]]。\n\n"
|
||||
"[[table:Q3 cols=日期,门店,订单数,到店人数 rows=0-3]]")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 夹具
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def engine_up():
|
||||
await run_manager.setup()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
async def _source(name: str, path: Path, options: dict | None = None) -> str:
|
||||
async with SessionLocal() as session:
|
||||
row = (await session.execute(select(DataSource).where(DataSource.name == name))).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True)
|
||||
session.add(row)
|
||||
row.database, row.kind, row.enabled, row.readonly, row.options = str(path), "sqlite", True, True, options or {}
|
||||
await session.commit()
|
||||
return row.id
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def sources(tmp_path, engine_up):
|
||||
"""把两个夹具脚本建成两个独立的 SQLite 库,各接成一个只读数据源。"""
|
||||
ids = []
|
||||
for name in ("stores", "members"):
|
||||
path = tmp_path / f"{name}.db"
|
||||
db = sqlite3.connect(path)
|
||||
db.executescript((FIXTURES / f"{name}.sql").read_text("utf-8"))
|
||||
db.commit()
|
||||
db.close()
|
||||
ids.append(await _source(f"merge_{name}", path))
|
||||
for sid in ids:
|
||||
await engines.invalidate(sid)
|
||||
yield
|
||||
for sid in ids:
|
||||
await engines.invalidate(sid)
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||||
|
||||
|
||||
def edge(a, b):
|
||||
return {"source": a, "target": b}
|
||||
|
||||
|
||||
def store_graph(*, sales_sql=SALES_SQL, visits_sql=VISITS_SQL, merge_sql=MERGE_SQL, visits_args=None, report=True,
|
||||
inputs=None):
|
||||
"""入口分两路查两个库,汇合到合并查询,再接口径卡、报告、成果。"""
|
||||
nodes = [
|
||||
node("start", "input"),
|
||||
node("q_sales", "tool", tool="db_query__merge_stores", args={"sql": sales_sql}),
|
||||
node("q_visits", "tool", tool="db_query__merge_members", args={"sql": visits_sql, **(visits_args or {})}),
|
||||
node("merge", "merge", inputs=inputs or {"s": "q_sales", "v": "q_visits"}, sql=merge_sql),
|
||||
node("card", "metrics", caliber="门店经营", caliber_version="v1", metrics=[
|
||||
{"id": "conversion", "name": "转化率", "decimals": 4, "format": "percent_of_ratio",
|
||||
"expression": "cell(nodes.merge, 0, '订单数') / cell(nodes.merge, 0, '到店人数')"},
|
||||
{"id": "aov", "name": "客单价", "unit": "元", "decimals": 1,
|
||||
"expression": "cell(nodes.merge, 0, '销售额') / cell(nodes.merge, 0, '订单数')"}]),
|
||||
]
|
||||
edges = [edge("start", "q_sales"), edge("start", "q_visits"), edge("q_sales", "merge"), edge("q_visits", "merge"),
|
||||
edge("merge", "card")]
|
||||
if report:
|
||||
nodes += [node("write", "report", instructions="写门店日报", on_violation="flag"),
|
||||
node("out", "output", fields=[{"name": "日报", "value": "{{ nodes.write.text }}"}])]
|
||||
edges += [edge("card", "write"), edge("write", "out")]
|
||||
else:
|
||||
nodes.append(node("out", "output", fields=[{"name": "转化率", "value": "{{ nodes.card.metrics }}"}]))
|
||||
edges.append(edge("card", "out"))
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
|
||||
|
||||
def script(monkeypatch, text=TEXT):
|
||||
from app.providers import mock_model
|
||||
|
||||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text))
|
||||
|
||||
|
||||
async def wait(run_id: str) -> Run:
|
||||
for _ in range(400):
|
||||
await asyncio.sleep(0.05)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run_id)
|
||||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||||
return row
|
||||
raise AssertionError(f"run {run_id} 没有结束:{row.status}")
|
||||
|
||||
|
||||
async def run_graph(monkeypatch, graph=None, text=TEXT) -> Run:
|
||||
script(monkeypatch, text)
|
||||
return await wait((await run_manager.start(graph=graph or store_graph(), input_payload={})).id)
|
||||
|
||||
|
||||
async def events(run_id: str, etype: str, node_id: str | None = None) -> list[dict]:
|
||||
async with SessionLocal() as session:
|
||||
query = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype)
|
||||
if node_id:
|
||||
query = query.where(RunEvent.node_id == node_id)
|
||||
return [e.data for e in (await session.execute(query.order_by(RunEvent.seq))).scalars()]
|
||||
|
||||
|
||||
def load_doc(row: Run) -> dict:
|
||||
doc_id = row.output["_evidence"]["doc_artifact"]
|
||||
return json.loads((settings.data_dir / "artifacts" / doc_id[:2] / f"{doc_id}.json").read_text("utf-8"))
|
||||
|
||||
|
||||
def seg_of(doc: dict, text: str, n: int = 0) -> dict:
|
||||
return [s for s in iter_segments(doc) if s["text"] == text and s.get("cite")][n]
|
||||
|
||||
|
||||
def alias_of(doc: dict, node_id: str) -> str:
|
||||
[alias] = [a for a, e in doc["catalog"].items() if e.get("kind") == "query" and e.get("node_id") == node_id]
|
||||
return alias
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 画图时的校验
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def merge_spec(merge_cfg: dict, *, extra_nodes=(), extra_edges=()) -> GraphSpec:
|
||||
base = store_graph(report=False)
|
||||
nodes = [n if n["id"] != "merge" else node("merge", "merge", **merge_cfg) for n in base["nodes"]]
|
||||
return GraphSpec.model_validate({"nodes": [*nodes, *extra_nodes], "edges": [*base["edges"], *extra_edges]})
|
||||
|
||||
|
||||
def merge_errors(spec: GraphSpec) -> list:
|
||||
return [i for i in validate_graph(spec).issues if i.node_id == "merge" and i.level == "error"]
|
||||
|
||||
|
||||
def test_node_type_exists():
|
||||
assert NodeType.MERGE == "merge"
|
||||
|
||||
|
||||
def test_a_well_formed_merge_validates():
|
||||
spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL})
|
||||
assert merge_errors(spec) == []
|
||||
|
||||
|
||||
def test_merge_of_a_merge_is_allowed():
|
||||
spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL},
|
||||
extra_nodes=[node("again", "merge", inputs={"m": "merge"}, sql="SELECT * FROM m")],
|
||||
extra_edges=[edge("merge", "again")])
|
||||
assert [i for i in validate_graph(spec).issues if i.node_id == "again" and i.level == "error"] == []
|
||||
|
||||
|
||||
def test_inputs_are_required():
|
||||
[issue] = merge_errors(merge_spec({"inputs": {}, "sql": MERGE_SQL}))
|
||||
assert issue.field == "inputs" and issue.code == "merge.inputs_missing"
|
||||
|
||||
|
||||
def test_input_pointing_to_a_missing_node():
|
||||
[issue] = merge_errors(merge_spec({"inputs": {"s": "q_sales", "v": "nope"}, "sql": MERGE_SQL}))
|
||||
assert issue.field == "inputs.v" and "不存在" in issue.message and issue.code == "merge.input_invalid"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("extra, why", [
|
||||
(node("think", "llm", prompt="写点什么"), "不产出查询结果"),
|
||||
(node("calc", "tool", tool="calculator", args={"expression": "1+1"}), "不是数据库查询工具"),
|
||||
(node("asker", "agent", prompt="查一下", tools=["db_query__merge_stores"]), "Agent"),
|
||||
])
|
||||
def test_input_must_produce_a_query_result(extra, why):
|
||||
spec = merge_spec({"inputs": {"s": "q_sales", "v": extra["id"]}, "sql": MERGE_SQL},
|
||||
extra_nodes=[extra], extra_edges=[edge("start", extra["id"]), edge(extra["id"], "merge")])
|
||||
[issue] = merge_errors(spec)
|
||||
assert issue.field == "inputs.v" and why in issue.message and issue.code == "merge.input_invalid"
|
||||
|
||||
|
||||
def test_agent_inputs_explain_how_to_fix():
|
||||
extra = node("asker", "agent", prompt="查一下", tools=["db_query__merge_stores"])
|
||||
spec = merge_spec({"inputs": {"s": "q_sales", "v": "asker"}, "sql": MERGE_SQL},
|
||||
extra_nodes=[extra], extra_edges=[edge("start", "asker"), edge("asker", "merge")])
|
||||
[issue] = merge_errors(spec)
|
||||
assert "查询多次" in issue.message and "调用工具" in issue.message
|
||||
|
||||
|
||||
def test_input_must_be_upstream():
|
||||
lonely = node("q_other", "tool", tool="db_query__merge_stores", args={"sql": "SELECT 1 AS x"})
|
||||
spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_other"}, "sql": MERGE_SQL},
|
||||
extra_nodes=[lonely], extra_edges=[edge("start", "q_other")])
|
||||
[issue] = merge_errors(spec)
|
||||
assert issue.field == "inputs.v" and "上游" in issue.message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alias", ["1s", "a-b", "select", "sqlite_x"])
|
||||
def test_alias_must_be_a_legal_table_name(alias):
|
||||
[issue] = merge_errors(merge_spec({"inputs": {"s": "q_sales", alias: "q_visits"}, "sql": MERGE_SQL}))
|
||||
assert issue.field == f"inputs.{alias}" and issue.code == "merge.alias_invalid" and alias in issue.message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, why", [
|
||||
("", "还没有填写"),
|
||||
("DELETE FROM s", "SELECT 或 WITH"),
|
||||
("PRAGMA table_info(s)", "SELECT 或 WITH"),
|
||||
("SELECT * FROM s; SELECT * FROM v", "一条"),
|
||||
])
|
||||
def test_sql_must_be_one_select_or_with(sql, why):
|
||||
[issue] = merge_errors(merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": sql}))
|
||||
assert issue.field == "sql" and why in issue.message and issue.code == "merge.sql_invalid"
|
||||
|
||||
|
||||
def test_merge_inputs_count_as_references_to_the_upstream_nodes():
|
||||
from app.engine.variables import analyze
|
||||
|
||||
report = analyze(merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL}))
|
||||
refs = {v.path: v.refs for v in report.variables}
|
||||
assert any(r.node_id == "merge" for r in refs["nodes.q_sales"])
|
||||
|
||||
|
||||
def test_labels_and_governance_know_the_type():
|
||||
from app.engine.governance import _traceable
|
||||
from app.engine.labels import field_label, type_label
|
||||
from app.engine.upgrade import _queries
|
||||
|
||||
spec = merge_spec({"inputs": {"s": "q_sales", "v": "q_visits"}, "sql": MERGE_SQL})
|
||||
merge = spec.node_map()["merge"]
|
||||
assert type_label("merge") == "合并查询"
|
||||
assert field_label("sql", "merge") == "合并 SQL" and field_label("inputs", "merge") == "输入"
|
||||
assert _traceable(merge) and _queries(merge)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 执行与证据链
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_end_to_end_two_databases(monkeypatch, sources, client):
|
||||
row = await run_graph(monkeypatch)
|
||||
assert row.status == "succeeded", row.error
|
||||
|
||||
[finished] = await events(row.id, "node.finished", "merge")
|
||||
output = artifact_store.load(finished["artifact"])
|
||||
assert output["columns"] == ["日期", "门店", "订单数", "销售额", "到店人数"]
|
||||
assert output["rows"] == [["2026-05-01", "S01", 3, 300, 6], ["2026-05-01", "S02", 2, 150, 4],
|
||||
["2026-05-02", "S01", 2, 250, 5], ["2026-05-02", "S02", 1, 75, 4]]
|
||||
assert output["source"] == MERGE_SOURCE and output["warnings"] == []
|
||||
|
||||
# 台账:一条和数据源查询同形的 query 条目,报告据此编号
|
||||
[entry] = finished["evidence"]
|
||||
assert entry["kind"] == "query" and entry["source"] == MERGE_SOURCE and entry["artifact"] == output["artifact"]
|
||||
assert entry["rows"] == 4 and entry["truncated"] is False
|
||||
snapshot = artifact_store.load(entry["artifact"])
|
||||
assert [i["alias"] for i in snapshot["inputs"]] == ["s", "v"]
|
||||
assert snapshot["sources"] == ["merge_stores", "merge_members"]
|
||||
assert snapshot["sql"] == MERGE_SQL
|
||||
|
||||
# 运行事件:合并的输入、SQL、行数,运行详情据此展示
|
||||
[done] = await events(row.id, "merge.end", "merge")
|
||||
assert done["sql"] == MERGE_SQL and done["rows"] == 4 and done["warnings"] == []
|
||||
assert [(i["alias"], i["node_id"], i["rows"], i["source"]) for i in done["inputs"]] == [
|
||||
("s", "q_sales", 4, "merge_stores"), ("v", "q_visits", 4, "merge_members")]
|
||||
assert done["query_artifact"] == entry["artifact"]
|
||||
|
||||
doc = load_doc(row)
|
||||
q3 = doc["catalog"]["Q3"]
|
||||
assert q3["node_id"] == "merge" and q3["source"] == MERGE_SOURCE and q3["artifact"] == entry["artifact"]
|
||||
assert {alias_of(doc, "q_sales"), alias_of(doc, "q_visits")} == {"Q1", "Q2"}
|
||||
metric = doc["catalog"]["m:conversion"]
|
||||
assert metric["value"] == 0.5 and metric["rendered"] == "50.00%"
|
||||
# 口径卡的两个输入都按 cell() 落在合并结果的那一格上,并按快照核对过
|
||||
card = artifact_store.load(metric["artifact"])
|
||||
conversion = next(m for m in card["metrics"] if m["id"] == "conversion")
|
||||
assert [(i["via"], i["status"], i["artifact"], i["locator"]) for i in conversion["inputs"]] == [
|
||||
("tool_cell", "verified", entry["artifact"], {"row": 0, "column": "订单数"}),
|
||||
("tool_cell", "verified", entry["artifact"], {"row": 0, "column": "到店人数"})]
|
||||
text = row.output["日报"]
|
||||
assert "到店 6 人" in text and "50.00%" in text and "100.0元" in text
|
||||
|
||||
|
||||
async def test_cell_segment_traces_to_both_original_queries(monkeypatch, sources, client):
|
||||
row = await run_graph(monkeypatch)
|
||||
assert row.status == "succeeded", row.error
|
||||
doc = load_doc(row)
|
||||
qv = alias_of(doc, "q_visits")
|
||||
qs = alias_of(doc, "q_sales")
|
||||
seg = seg_of(doc, "6")
|
||||
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}")).json()
|
||||
merge_step, visits_step = body["chain"]
|
||||
|
||||
assert merge_step["step"] == "query" and merge_step["alias"] == "Q3" and merge_step["source"] == MERGE_SOURCE
|
||||
assert merge_step["hash_ok"] is True and merge_step["sealed"] is True
|
||||
assert "mask_note" not in merge_step # 合并查询不是数据源,不能说它「已不存在」
|
||||
merged = merge_step["merge"]
|
||||
assert merged["sql"] == MERGE_SQL and merged["warnings"] == []
|
||||
assert [(i["alias"], i["node_id"], i["query"], i["rows"], i["source"], i["sealed"]) for i in merged["inputs"]] == [
|
||||
("s", "q_sales", qs, 4, "merge_stores", True), ("v", "q_visits", qv, 4, "merge_members", True)]
|
||||
assert merged["traced"] == [{"cell": [0, "到店人数"], "input": "v", "query": qv, "row": 2, "column": "到店人数"}]
|
||||
|
||||
# 接着给出会员库那次查询,高亮的正是那一格,值相同
|
||||
assert visits_step["step"] == "query" and visits_step["alias"] == qv and visits_step["merged_into"] == "Q3"
|
||||
assert visits_step["highlight"]["cells"] == [[2, "到店人数"]]
|
||||
at = visits_step["row_index"].index(2)
|
||||
assert visits_step["rows"][at][visits_step["columns"].index("到店人数")] == 6
|
||||
|
||||
|
||||
async def test_metric_segment_traces_through_the_merge(monkeypatch, sources, client):
|
||||
row = await run_graph(monkeypatch)
|
||||
doc = load_doc(row)
|
||||
qs, qv = alias_of(doc, "q_sales"), alias_of(doc, "q_visits")
|
||||
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_of(doc, '50.00%')['id']}")).json()
|
||||
metric, *inputs = body["chain"][:3]
|
||||
assert metric["step"] == "metric" and metric["value"] == 0.5
|
||||
assert [(i["via"], i["status"], i["query"], i["cell"]) for i in inputs] == [
|
||||
("tool_cell", "verified", "Q3", "Q3.r0.订单数"), ("tool_cell", "verified", "Q3", "Q3.r0.到店人数")]
|
||||
steps = [s for s in body["chain"] if s["step"] == "query"]
|
||||
assert [s["alias"] for s in steps] == ["Q3", qs, qv]
|
||||
merge_step, sales_step, visits_step = steps
|
||||
assert sorted(t["input"] for t in merge_step["merge"]["traced"]) == ["s", "v"]
|
||||
assert sales_step["highlight"]["cells"] == [[0, "订单数"]] and sales_step["merged_into"] == "Q3"
|
||||
assert visits_step["highlight"]["cells"] == [[2, "到店人数"]]
|
||||
|
||||
|
||||
async def test_provenance_of_a_merged_cell_follows_the_trace(monkeypatch, sources, client):
|
||||
"""手工源上的格:追到会员库那次查询,结论和直接引用那一格相同(not_upload),并记下经过的合并。"""
|
||||
row = await run_graph(monkeypatch)
|
||||
doc = load_doc(row)
|
||||
seg = seg_of(doc, "6")
|
||||
out = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}/provenance")).json()
|
||||
assert (out["status"], out["reason"]["code"]) == ("none", "not_upload")
|
||||
assert out["cell"] == {"alias": "Q3", "row": 0, "column": "到店人数", "artifact": doc["catalog"]["Q3"]["artifact"]}
|
||||
assert out["merge"] == [{"alias": "Q3", "node_id": "merge", "input": "v", "query": alias_of(doc, "q_visits"),
|
||||
"row": 2, "column": "到店人数"}]
|
||||
|
||||
|
||||
async def test_a_computed_merge_column_has_no_cell_lineage(monkeypatch, sources, client):
|
||||
sql = ("SELECT s.日期, s.门店, s.订单数 + v.到店人数 AS 合计 FROM s JOIN v ON s.日期 = v.日期 AND s.门店 = v.门店 "
|
||||
"ORDER BY s.日期, s.门店")
|
||||
graph = store_graph(merge_sql=sql)
|
||||
graph["nodes"] = [n for n in graph["nodes"] if n["id"] != "card"]
|
||||
graph["edges"] = [e for e in graph["edges"] if "card" not in (e["source"], e["target"])] + [edge("merge", "write")]
|
||||
row = await run_graph(monkeypatch, graph, text="合计 [[v:Q3.r0.合计]]。")
|
||||
assert row.status == "succeeded", row.error
|
||||
doc = load_doc(row)
|
||||
seg = seg_of(doc, "9")
|
||||
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}")).json()
|
||||
[merge_step] = body["chain"]
|
||||
[traced] = merge_step["merge"]["traced"]
|
||||
assert traced["input"] is None and traced["query"] is None and "逐格来历" in traced["note"]
|
||||
out = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}/provenance")).json()
|
||||
assert (out["status"], out["reason"]["code"], out["version"]) == ("none", "merge_no_lineage", None)
|
||||
assert out["merge"] == [{"alias": "Q3", "node_id": "merge", "input": None, "query": None, "row": None,
|
||||
"column": None}]
|
||||
|
||||
|
||||
async def test_a_truncated_input_fails_the_node(monkeypatch, sources):
|
||||
row = await run_graph(monkeypatch, store_graph(visits_args={"limit": 2}, report=False))
|
||||
assert row.status == "failed"
|
||||
[failed] = await events(row.id, "node.failed", "merge")
|
||||
assert "「v」" in failed["error"] and "截断" in failed["error"] and "聚合" in failed["error"]
|
||||
|
||||
|
||||
async def test_warnings_reach_the_output_and_the_events(monkeypatch, sources):
|
||||
"""门店编号一边是文本 '01'、一边是数 1;只按门店合并、没按日期:两道警告都要看得见。"""
|
||||
sales_sql = ("SELECT order_date AS 日期, substr(store_id, 2) AS 门店, COUNT(*) AS 订单数, SUM(amount) AS 销售额 "
|
||||
"FROM orders GROUP BY order_date, store_id ORDER BY 1, 2")
|
||||
visits_sql = ("SELECT visit_date AS 日期, CAST(substr(store_code, 2) AS INTEGER) AS 门店, COUNT(*) AS 到店人数 "
|
||||
"FROM visits GROUP BY visit_date, store_code ORDER BY 1, 2")
|
||||
merge_sql = "SELECT s.门店, s.订单数, s.销售额, v.到店人数 FROM s JOIN v ON s.门店 = v.门店"
|
||||
row = await run_graph(monkeypatch, store_graph(sales_sql=sales_sql, visits_sql=visits_sql, merge_sql=merge_sql,
|
||||
report=False))
|
||||
assert row.status == "succeeded", row.error
|
||||
[finished] = await events(row.id, "node.finished", "merge")
|
||||
output = artifact_store.load(finished["artifact"])
|
||||
assert sorted(w["code"] for w in output["warnings"]) == ["key_type_mismatch", "rows_grew"]
|
||||
[done] = await events(row.id, "merge.end", "merge")
|
||||
assert sorted(w["code"] for w in done["warnings"]) == ["key_type_mismatch", "rows_grew"]
|
||||
logs = [e for e in await events(row.id, "log", "merge") if e.get("level") == "warn"]
|
||||
assert sorted(e["code"] for e in logs) == ["merge_key_type", "merge_rows_grew"]
|
||||
assert all(e["message"] for e in logs)
|
||||
|
||||
|
||||
async def test_rerunning_the_node_on_the_same_inputs_gives_the_same_snapshot(monkeypatch, sources):
|
||||
"""重放、继续运行时节点从头再执行:同样的输入得到同一份快照、同一条台账,下游的出处不变。"""
|
||||
from app.engine.context import NodeContext, RunContext
|
||||
from app.engine.nodes.merge import run_merge
|
||||
|
||||
row = await run_graph(monkeypatch, store_graph(report=False))
|
||||
assert row.status == "succeeded", row.error
|
||||
outputs = {}
|
||||
for nid in ("q_sales", "q_visits"):
|
||||
[finished] = await events(row.id, "node.finished", nid)
|
||||
outputs[nid] = artifact_store.load(finished["artifact"])
|
||||
spec = GraphSpec.model_validate(store_graph(report=False))
|
||||
ctx = NodeContext(node=spec.node_map()["merge"],
|
||||
run=RunContext(run_id=row.id, thread_id=row.id, spec=spec, agent_limits={}))
|
||||
state = {"nodes": outputs, "trail": [], "evidence": []}
|
||||
first, second = await run_merge(state, ctx), await run_merge(state, ctx)
|
||||
assert first == second
|
||||
assert first["evidence"][0]["artifact"] == first["nodes"]["merge"]["artifact"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 助手
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_node_reference_explains_when_to_use_merge():
|
||||
from app.api.copilot import NODE_REFERENCE
|
||||
|
||||
assert "- merge:" in NODE_REFERENCE
|
||||
block = NODE_REFERENCE.split("- merge:", 1)[1].split("\n- ", 1)[0]
|
||||
for phrase in ("inputs", "sql", "不同的库", "同一个库", "一条 SQL", "口径卡", "截断", "cell(nodes."):
|
||||
assert phrase in block, phrase
|
||||
Reference in New Issue
Block a user