mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 14:21:58 +00:00
ThearchyHelios
fcdcc30d4e
一、可点击证据第二期
取数链路
- 智能体、工具节点、代码节点、知识检索取数时都记进证据台账；报告目录按台账编出 Q1… / K1…
- 报告可以引用快照里的一格（[[v:Q1.r0.gmv]]），也可以由系统从快照生成整张表（[[table:Q1 cols=… rows=0-4]]），
表里每一格都是可点的片段；取值一律以快照为准，取回快照时复验哈希
- 口径卡表达式新增 cell(x, row, column)，lineage 精确到单元格
- 智能体开 cite_fields（配合 output_schema）后，循环结束做一次结构化抽取，每个字段标出取自哪次查询的哪一格，
再到快照里核对：一致记 verified；不一致以快照为准并告警；查不到的记为空值，永远不兜底成 0
- 代码节点新增 evidence_role（source / compute）和 code_sha；沙箱算出来的数不能被报告直接引用，要进口径卡
- 查询快照记下列类型，文本列按文本渲染（年份 "2026" 不再显示成「2,026」）
- 升级前发起的运行（停在审批上的）一律走旧逻辑：新的 task 只追加在循环之后，恢复时不会错号
证据接口与面板
- 片段链接上查询步骤：SQL、被引用的行加前后各 2 行、高亮到格；数据源可设 mask_columns 遮罩列，
遮罩在查询时记进快照，数据源事后改名或删除也照样遮罩（在有身份体系之前，遮罩只减少暴露，不是安全边界）
- 输入来源写明核对结果（与快照一致 / 模型报的值和快照不一致 / 没查到记为空）；口径卡缺输入时也给出完整的链
- 受管出具要在契约里声明 cells 才能引用单元格；受管与否记在所钉版本上，改画布不再让受管版本的正式运行降级
- 口径卡可以用 caliber_from 钉住另一个已发布工作流里的口径卡，上游有新版本时按 upgrade_policy 处置，
和子工作流同一套规则；子工作流节点也补上了升版处置的配置项
Copilot、校验与报错
- 问数据的答案里有数字时，搭 input → 智能体 → 报告 → 出口，报告直接引用查过的单元格
- 写 SQL 前核对数据源的表结构，SQL 里出现表结构里没有的列会被打回
- 不再搭「智能体文字 → 转换节点解析 JSON」这种图；转换节点的 JSON 报错分清是上游模型写坏了、
模板里裸插了模型文字（给出 | json 的改法），还是模板本身写错了
- 工具节点遇到「查询失败」「SQL 被拒绝」判为失败；发起运行前检查绑定的工具是否存在，不存在时直接拒绝并给出入口
二、发布前自动修复
- 问题带上机器编号（code）和修复 id；新增只读接口 POST /api/workflows/{id}/publish-check 和 /autofix
- 修复分三类：答案唯一、不降低要求的一键修复；要人拿主意的给选项（不替人选）；结构性问题交给 Copilot
- 一律先预览，不自动保存、不自动发布；删节点、删契约、放宽审批、关掉 strict、改级别等降低要求的改动一律拒绝，
Copilot 的方案也要过这一关；应用后重跑检查，错误必须变少且不能冒出新的
- 发布弹窗打开就先检查；问题面板新增「发布前检查」，平时就能看到、就地修掉
- 门禁修正：契约有 report_from 时不再要求 narrative（模板⑨可以发成受管）；全图默认审批为「全部放行」时同样拦下
- 修掉画布保存会抹掉全图默认（defaults）的老问题：以前任何一次保存都会把它清空，审批 always 因此被悄悄放宽
验证（沙箱）：tsc 干净，生产构建通过，pytest 2268 passed（两遍），check-all 17 项检查、3915 项全过
（新增 check-publish）；关键行为做了变异验证、全部抓到。
489 lines
24 KiB
Python
489 lines
24 KiB
Python
"""取数链路进证据台账：tool 节点、agent、知识检索都记账，口径卡能精确到单元格。
|
||
|
||
一期的台账里只有口径卡的指标集：报告点开一个数，最远追到「vars.q.rows[0][0]」这条
|
||
路径，追不到那次查询的哪一格。二期：
|
||
|
||
- tool 节点落 tool 条目，查库再落 query 条目（工件是查询快照，外面包着工具快照）
|
||
- agent 每次查库都记账，给模型看的工具结果前面标上本节点内的编号「【证据 Q1】」
|
||
- 知识检索落 retrieval 条目
|
||
- 报告目录按全局台账顺序编出 Q1… / K1…
|
||
- 口径卡用 cell(nodes.fetch, 0, "amount") 取数时，来历精确到那一格（via tool_cell）
|
||
- 数据源工具返回「查询失败：…」「SQL 被拒绝：…」时，tool 节点判失败，报错写原话
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import sqlite3
|
||
|
||
import pytest
|
||
from langchain_core.messages import AIMessage, ToolMessage
|
||
from sqlalchemy import select
|
||
|
||
from app.core import artifact_store
|
||
from app.data.engine import engines
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import DataSource, Run, RunEvent
|
||
from app.engine.evidence import build_catalog, make_eid
|
||
from app.engine.expressions import (
|
||
CellError,
|
||
ExpressionError,
|
||
eval_expression,
|
||
leaf_refs,
|
||
parse_expression,
|
||
substitute,
|
||
table_cell,
|
||
)
|
||
from app.engine.runner import run_manager
|
||
|
||
TOOL = "db_query__shop"
|
||
SQL = "SELECT SUM(amount) AS gmv, SUM(prev) AS gmv_prev, COUNT(*) AS orders FROM orders WHERE week = '2026-W37'"
|
||
|
||
|
||
@pytest.fixture
|
||
async def engine_up():
|
||
await run_manager.setup()
|
||
yield
|
||
await run_manager.shutdown()
|
||
|
||
|
||
@pytest.fixture
|
||
async def shop(tmp_path):
|
||
"""示例 shop 库：本周 3 单，金额 100.5 + 200 + 300。"""
|
||
path = tmp_path / "shop.db"
|
||
db = sqlite3.connect(path)
|
||
db.executescript("CREATE TABLE orders (id INTEGER PRIMARY KEY, week TEXT, amount REAL, prev REAL, refunded INTEGER);")
|
||
db.executemany("INSERT INTO orders VALUES (?, ?, ?, ?, ?)", [
|
||
(1, "2026-W37", 100.5, 90.0, 0), (2, "2026-W37", 200.0, 100.0, 1), (3, "2026-W37", 300.0, 110.0, 0),
|
||
(4, "2026-W36", 50.0, 40.0, 0)])
|
||
db.commit()
|
||
db.close()
|
||
async with SessionLocal() as session:
|
||
row = (await session.execute(select(DataSource).where(DataSource.name == "shop"))).scalar_one_or_none()
|
||
if row is None:
|
||
row = DataSource(name="shop", kind="sqlite", database=str(path), readonly=True, enabled=True)
|
||
session.add(row)
|
||
else:
|
||
row.database, row.kind, row.enabled, row.readonly = str(path), "sqlite", True, True
|
||
await session.commit()
|
||
source_id = row.id
|
||
# 连接池按数据源 id 缓存：换了库文件不作废，查的还是上一个测试的库
|
||
await engines.invalidate(source_id)
|
||
yield
|
||
await engines.invalidate(source_id)
|
||
|
||
|
||
def node(nid, ntype, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||
|
||
|
||
def chain(*nodes):
|
||
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||
|
||
|
||
async def finish(graph, statuses=("succeeded", "failed")) -> Run:
|
||
run = await run_manager.start(graph=graph, input_payload={})
|
||
for _ in range(400):
|
||
await asyncio.sleep(0.05)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Run, run.id)
|
||
if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None):
|
||
return row
|
||
raise AssertionError(f"run {run.id} 没跑完：{row.status}")
|
||
|
||
|
||
async def events(run_id: str, etype: str | None = None, node_id: str | None = None) -> list[RunEvent]:
|
||
async with SessionLocal() as session:
|
||
q = select(RunEvent).where(RunEvent.run_id == run_id)
|
||
if etype:
|
||
q = q.where(RunEvent.type == etype)
|
||
if node_id:
|
||
q = q.where(RunEvent.node_id == node_id)
|
||
return list((await session.execute(q.order_by(RunEvent.seq))).scalars())
|
||
|
||
|
||
async def ledger(run_id: str) -> list[dict]:
|
||
tup = await run_manager.checkpointer.aget_tuple({"configurable": {"thread_id": run_id}})
|
||
return list(tup.checkpoint["channel_values"].get("evidence") or [])
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# cell()：纯函数
|
||
# --------------------------------------------------------------------------
|
||
|
||
PAYLOAD = {"columns": ["gmv", "orders"], "rows": [[600.5, 3], ["45678.50", 7]], "artifact": "a" * 64}
|
||
|
||
|
||
def test_cell_reads_json_text_and_objects():
|
||
text = json.dumps(PAYLOAD)
|
||
assert table_cell(text, 0, "gmv") == 600.5
|
||
assert table_cell(PAYLOAD, 0, 1) == 3 # 列号也认
|
||
assert table_cell(PAYLOAD, 1, "gmv") == 45678.5 # DECIMAL 字符串能参与运算
|
||
assert table_cell({"rows": [{"gmv": 1}]}, 0, "gmv") == 1
|
||
assert eval_expression("cell(nodes.q, 0, 'gmv') * 2", {"nodes": {"q": text}}) == 1201.0
|
||
|
||
|
||
@pytest.mark.parametrize("args, words", [
|
||
((PAYLOAD, 2, "gmv"), "只有 2 行"),
|
||
((PAYLOAD, 0, "amount"), "没有列「amount」"),
|
||
((PAYLOAD, 0, 5), "只有 2 列"),
|
||
((PAYLOAD, -1, "gmv"), "从 0 数"),
|
||
((PAYLOAD, True, "gmv"), "整数"),
|
||
(("查询失败：no such table: orders", 0, "gmv"), "查询失败：no such table: orders"),
|
||
(({"columns": ["gmv"], "rows": []}, 0, "gmv"), "0 行"),
|
||
((None, 0, "gmv"), "查询结果"),
|
||
])
|
||
def test_cell_errors_are_readable(args, words):
|
||
with pytest.raises(CellError) as info:
|
||
table_cell(*args)
|
||
assert words in str(info.value) and isinstance(info.value, ExpressionError)
|
||
assert str(info.value).startswith("cell()")
|
||
|
||
|
||
# 驱动把 DECIMAL 一律落成 str(Decimal)：小数位为 0 的——MySQL 对整数列 SUM()（包括
|
||
# SUM(CASE WHEN … THEN 1 ELSE 0 END) 这种计数写法）、Postgres 的 SUM(bigint)——是没有小数点的
|
||
# "45678"；很小的数写成 "1E-7"
|
||
INT_DECIMAL = {"columns": ["gmv", "refunds", "code", "tiny"], "rows": [["45678", "29", "00123", "1E-7"]],
|
||
"artifact": "a" * 64}
|
||
|
||
|
||
def test_cell_treats_integer_decimal_text_as_numbers():
|
||
gmv = table_cell(INT_DECIMAL, 0, "gmv")
|
||
assert gmv == 45678 and isinstance(gmv, int)
|
||
assert table_cell(INT_DECIMAL, 0, "refunds") == 29
|
||
assert table_cell(INT_DECIMAL, 0, "tiny") == 1e-7
|
||
assert table_cell(INT_DECIMAL, 0, "code") == "00123" # 带前导零的是编号，照文本
|
||
ctx = {"nodes": {"q": json.dumps(INT_DECIMAL)}}
|
||
assert eval_expression("cell(nodes.q, 0, 'gmv') / 2", ctx) == 22839.0
|
||
assert eval_expression("cell(nodes.q, 0, 'refunds') + 1", ctx) == 30
|
||
|
||
|
||
def test_leaf_refs_treat_a_cell_call_as_one_input():
|
||
tree = parse_expression("round(cell(nodes.fetch, 0, 'gmv') / cell(nodes.fetch, vars.i, 'orders'), 2)")[0]
|
||
assert leaf_refs(tree) == ["cell(nodes.fetch, 0, 'gmv')", "cell(nodes.fetch, vars.i, 'orders')", "vars.i"]
|
||
text = substitute(tree, {"cell(nodes.fetch, 0, 'gmv')": 600.5, "cell(nodes.fetch, vars.i, 'orders')": 3,
|
||
"vars.i": 0})
|
||
assert text == "round(600.5 / 3, 2)"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# tool 节点
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def fetch_graph(*after, **tool_cfg):
|
||
return chain(
|
||
node("start", "input"),
|
||
node("fetch", "tool", tool=TOOL, args={"sql": SQL}, **tool_cfg),
|
||
*after,
|
||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.fetch }}"}]),
|
||
)
|
||
|
||
|
||
async def test_tool_node_writes_tool_and_query_entries(engine_up, shop):
|
||
row = await finish(fetch_graph())
|
||
assert row.status == "succeeded", row.error
|
||
[end] = await events(row.id, "tool.end", "fetch")
|
||
payload = json.loads(row.output["r"])
|
||
assert end.data["query_artifact"] == payload["artifact"]
|
||
[finished] = await events(row.id, "node.finished", "fetch")
|
||
tool_entry, query_entry = finished.data["evidence"]
|
||
assert tool_entry == {"kind": "tool", "node_id": "fetch", "exec": 1, "artifact": end.data["artifact"], "tool": TOOL}
|
||
assert query_entry == {"kind": "query", "node_id": "fetch", "exec": 1, "artifact": payload["artifact"],
|
||
"via": end.data["artifact"], "tool": TOOL, "source": "shop",
|
||
"columns": ["gmv", "gmv_prev", "orders"], "rows": 1, "truncated": False}
|
||
assert await ledger(row.id) == [tool_entry, query_entry]
|
||
snap = artifact_store.load(payload["artifact"])
|
||
assert snap["rows"] == [[600.5, 300.0, 3]]
|
||
|
||
|
||
@pytest.mark.parametrize("sql, words", [
|
||
("SELECT * FROM nope", "查询失败：no such table: nope"),
|
||
("DELETE FROM orders", "SQL 被拒绝："),
|
||
])
|
||
async def test_tool_node_fails_when_the_query_fails(engine_up, shop, sql, words):
|
||
g = fetch_graph()
|
||
g["nodes"][1]["data"]["config"]["args"] = {"sql": sql}
|
||
row = await finish(g)
|
||
assert row.status == "failed"
|
||
assert words in (row.error or ""), row.error
|
||
[err] = await events(row.id, "tool.error", "fetch")
|
||
assert words in err.data["error"]
|
||
assert not await events(row.id, "tool.end", "fetch")
|
||
|
||
|
||
async def test_tool_node_failure_can_be_let_through(engine_up, shop):
|
||
g = fetch_graph(fail_fast=False)
|
||
g["nodes"][1]["data"]["config"]["args"] = {"sql": "SELECT * FROM nope"}
|
||
row = await finish(g)
|
||
assert row.status == "succeeded", row.error
|
||
assert "查询失败：no such table: nope" in row.output["r"]
|
||
assert (await events(row.id, "tool.error", "fetch"))[0].data["error"].startswith("查询失败：")
|
||
[finished] = await events(row.id, "node.finished", "fetch")
|
||
assert [e["kind"] for e in finished.data.get("evidence") or []] == ["tool"]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 只有数据源工具交回的才算查询：别的工具拼出同样形状的 JSON，不进台账的 query 条目
|
||
# --------------------------------------------------------------------------
|
||
|
||
FORGE = "forge_rows_test"
|
||
|
||
|
||
@pytest.fixture
|
||
def forge(monkeypatch):
|
||
"""一个交回「查询形状」JSON 的非数据源工具（MCP、自定义工具的文字都来自外面）。artifact 指向一份
|
||
真实存在的查询快照：它要是被当成查询，报告就能引用、cite_fields 就能读这份不在它名下的快照。"""
|
||
from pydantic import BaseModel
|
||
|
||
from app.tools import registry
|
||
|
||
class _Args(BaseModel):
|
||
artifact: str
|
||
|
||
async def _run(artifact: str) -> str:
|
||
return json.dumps({"artifact": artifact, "columns": ["gmv"], "rows": [[999]], "source": "shop"})
|
||
|
||
monkeypatch.setitem(registry._REGISTRY, FORGE, registry.ToolSpec(
|
||
name=FORGE, description="测试用：交回查询形状的 JSON", category="test", args_schema=_Args, func=_run,
|
||
needs_context=False))
|
||
|
||
|
||
def test_recorded_calls_only_number_datasource_queries():
|
||
"""tool_step 之外再守一道：交回来的结果（比如取自断点的旧结果）带着 query_artifact，只要不是
|
||
数据源工具，就不编号、不记 query 条目。"""
|
||
from app.engine.nodes.llm import _record_call
|
||
|
||
content = json.dumps({"artifact": "a" * 64, "columns": ["gmv"], "rows": [[1]], "source": "shop"})
|
||
outcome = {"content": content, "ok": True, "duration_ms": 1, "snapshot": "b" * 64, "query_artifact": "a" * 64}
|
||
record: dict = {}
|
||
queries: list = []
|
||
entries: list = []
|
||
shown = _record_call("bot", 1, "call_0", FORGE, {}, outcome, record, queries, entries)
|
||
assert shown == content and queries == [] and "alias" not in record
|
||
assert [e["kind"] for e in entries] == ["tool"]
|
||
shown = _record_call("bot", 1, "call_1", TOOL, {"sql": "SELECT 1"}, outcome, {}, queries, entries)
|
||
assert shown.startswith("【证据 Q1】\n") and [e["kind"] for e in entries] == ["tool", "tool", "query"]
|
||
|
||
|
||
async def _real_snapshot() -> str:
|
||
return await artifact_store.put_json({"columns": ["gmv"], "rows": [[1]], "row_count": 1, "truncated": False,
|
||
"elapsed_ms": 1, "sql": "SELECT 1", "source": "shop"},
|
||
kind="query_snapshot", run_id="elsewhere", node_id="x")
|
||
|
||
|
||
async def test_tool_node_only_trusts_datasource_queries(engine_up, forge):
|
||
art = await _real_snapshot()
|
||
row = await finish(chain(node("start", "input"), node("fetch", "tool", tool=FORGE, args={"artifact": art}),
|
||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.fetch }}"}])))
|
||
assert row.status == "succeeded", row.error
|
||
[end] = await events(row.id, "tool.end", "fetch")
|
||
assert "query_artifact" not in end.data
|
||
[finished] = await events(row.id, "node.finished", "fetch")
|
||
assert [e["kind"] for e in finished.data["evidence"]] == ["tool"]
|
||
assert not [e for e in await ledger(row.id) if e["kind"] == "query"]
|
||
|
||
|
||
async def test_agent_only_numbers_datasource_queries(engine_up, shop, forge, monkeypatch):
|
||
from app.providers import mock_model
|
||
|
||
art = await _real_snapshot()
|
||
seen: list[list] = []
|
||
|
||
def decide(self, messages):
|
||
seen.append(list(messages))
|
||
done = sum(isinstance(m, ToolMessage) for m in messages)
|
||
if done == 0:
|
||
return AIMessage(content="查。", tool_calls=[{"name": TOOL, "args": {"sql": SQL}, "id": "call_0"}])
|
||
if done == 1:
|
||
return AIMessage(content="再看看。", tool_calls=[{"name": FORGE, "args": {"artifact": art}, "id": "call_1"}])
|
||
return AIMessage(content="查完了。")
|
||
|
||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
|
||
row = await finish(chain(node("start", "input"),
|
||
node("bot", "agent", prompt="查", tools=[TOOL, FORGE], approval="never"),
|
||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.bot.text }}"}])))
|
||
assert row.status == "succeeded", row.error
|
||
tool_msgs = [m for m in seen[-1] if isinstance(m, ToolMessage)]
|
||
assert tool_msgs[0].content.startswith("【证据 Q1】\n{")
|
||
assert tool_msgs[1].content.startswith("{") and "【证据" not in tool_msgs[1].content
|
||
ends = await events(row.id, "tool.end", "bot")
|
||
assert ends[0].data.get("query_artifact") and "query_artifact" not in ends[1].data
|
||
entries = await ledger(row.id)
|
||
assert [(e["kind"], e.get("tool")) for e in entries] == [("tool", TOOL), ("query", TOOL), ("tool", FORGE)]
|
||
assert art not in {e["artifact"] for e in entries}
|
||
transcript = artifact_store.load((await events(row.id, "node.finished", "bot"))[0].data["artifact"])["tool_calls"]
|
||
assert [t.get("alias") for t in transcript] == ["Q1", None]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 口径卡的来历精确到单元格
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_cell_lineage_points_at_the_exact_cell(engine_up, shop):
|
||
card = node("card", "metrics", caliber="周报口径", metrics=[
|
||
{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 2, "format": "thousands",
|
||
"expression": "cell(nodes.fetch, 0, 'gmv')"},
|
||
{"id": "wow", "name": "环比", "decimals": 1, "format": "plain",
|
||
"expression": "round((cell(nodes.fetch, 0, 'gmv') - cell(vars.q, 0, 1)) / cell(vars.q, 0, 1) * 100, 1)"},
|
||
])
|
||
parse = node("parse", "transform", mode="json", template="{{ nodes.fetch }}", assign_to="q")
|
||
row = await finish(fetch_graph(parse, card))
|
||
assert row.status == "succeeded", row.error
|
||
query = next(e for e in await ledger(row.id) if e["kind"] == "query")
|
||
finished = (await events(row.id, "node.finished", "card"))[0]
|
||
metrics = {m["id"]: m for m in artifact_store.load(finished.data["artifact"])["metrics"]}
|
||
|
||
gmv = metrics["gmv"]
|
||
assert gmv["value"] == 600.5 and gmv["substituted"] == "600.5" and gmv["recompute_ok"] is True
|
||
[cell] = gmv["inputs"]
|
||
assert cell == {"path": "cell(nodes.fetch, 0, 'gmv')", "value": 600.5, "node_id": "fetch", "via": "tool_cell",
|
||
"status": "verified", "locator": {"row": 0, "column": "gmv"}, "artifact": query["artifact"],
|
||
"eid": make_eid("cell", query["artifact"], {"row": 0, "column": "gmv"})}
|
||
|
||
# 经 transform 解析过的对象照样认得出是哪次查询；列号换成列名，eid 和按列名写的一致
|
||
prev = next(i for i in metrics["wow"]["inputs"] if i["path"] == "cell(vars.q, 0, 1)")
|
||
assert prev["via"] == "tool_cell" and prev["node_id"] == "fetch" and prev["status"] == "verified"
|
||
assert prev["locator"] == {"row": 0, "column": "gmv_prev"}
|
||
assert prev["eid"] == make_eid("cell", query["artifact"], {"row": 0, "column": "gmv_prev"})
|
||
|
||
|
||
async def test_card_computes_on_integer_decimal_text(engine_up, shop):
|
||
"""sqlite 没有 DECIMAL：CAST(… AS TEXT) 落进快照的，和 MySQL 整数列 SUM() 的 str(Decimal) 一模一样。"""
|
||
sql = ("SELECT CAST(SUM(refunded) AS TEXT) AS refunds, CAST(COUNT(*) AS TEXT) AS orders "
|
||
"FROM orders WHERE week = '2026-W37'")
|
||
card = node("card", "metrics", caliber="周报口径", metrics=[
|
||
{"id": "rate", "name": "退款率", "decimals": 1, "format": "plain",
|
||
"expression": "cell(nodes.fetch, 0, 'refunds') / cell(nodes.fetch, 0, 'orders') * 100"}])
|
||
g = fetch_graph(card)
|
||
g["nodes"][1]["data"]["config"]["args"] = {"sql": sql}
|
||
row = await finish(g)
|
||
assert row.status == "succeeded", row.error
|
||
assert json.loads(row.output["r"])["rows"] == [["1", "3"]]
|
||
finished = (await events(row.id, "node.finished", "card"))[0]
|
||
[rate] = artifact_store.load(finished.data["artifact"])["metrics"]
|
||
assert rate["value"] == pytest.approx(33.3) and rate["recompute_ok"] is True
|
||
assert {i["status"] for i in rate["inputs"]} == {"verified"}
|
||
assert [i["value"] for i in rate["inputs"]] == [1, 3]
|
||
|
||
|
||
async def test_cell_out_of_range_fails_the_card_readably(engine_up, shop):
|
||
card = node("card", "metrics", metrics=[{"id": "x", "expression": "cell(nodes.fetch, 3, 'gmv')"}])
|
||
row = await finish(fetch_graph(card))
|
||
assert row.status == "failed"
|
||
assert "cell() 要第 3 行，但这份查询结果只有 1 行" in (row.error or ""), row.error
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# agent 与知识检索也记账；目录按全局顺序编号
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def script_agent(monkeypatch, sqls):
|
||
from app.providers import mock_model
|
||
|
||
seen: list[list] = []
|
||
|
||
def decide(self, messages):
|
||
if self.response_format:
|
||
return AIMessage(content="{}")
|
||
seen.append(list(messages))
|
||
done = sum(isinstance(m, ToolMessage) for m in messages)
|
||
if done < len(sqls):
|
||
return AIMessage(content="查。", tool_calls=[{"name": TOOL, "args": {"sql": sqls[done]}, "id": f"call_{done}"}])
|
||
return AIMessage(content="查完了。")
|
||
|
||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
|
||
return seen
|
||
|
||
|
||
async def test_agent_and_retrieval_write_the_ledger_and_the_catalog_numbers_them(engine_up, shop, monkeypatch):
|
||
from app.memory import kb
|
||
|
||
async def fake_search(session, **kw):
|
||
return [{"title": "口径说明", "ordinal": 1, "content": "退款按发生周统计", "score": 0.9, "document_id": "d1"}]
|
||
|
||
monkeypatch.setattr(kb, "search", fake_search)
|
||
seen = script_agent(monkeypatch, [SQL, "SELECT * FROM nope", "SELECT COUNT(*) AS refunds FROM orders WHERE refunded = 1"])
|
||
g = chain(
|
||
node("start", "input"),
|
||
node("docs", "retrieve", query="退款口径", collection="docs"),
|
||
node("bot", "agent", prompt="查本周销售额和退款", tools=[TOOL], approval="never"),
|
||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.bot.text }}"}]),
|
||
)
|
||
row = await finish(g)
|
||
assert row.status == "succeeded", row.error
|
||
|
||
# 给模型看的：查成功的结果前面有本节点内的编号，失败的原话照旧喂回去、不编号
|
||
tool_msgs = [m for m in seen[-1] if isinstance(m, ToolMessage)]
|
||
assert tool_msgs[0].content.startswith("【证据 Q1】\n{")
|
||
assert tool_msgs[1].content.startswith("查询失败：no such table: nope")
|
||
assert tool_msgs[2].content.startswith("【证据 Q2】\n{")
|
||
|
||
entries = await ledger(row.id)
|
||
assert [(e["kind"], e["node_id"]) for e in entries] == [
|
||
("retrieval", "docs"), ("tool", "bot"), ("query", "bot"), ("tool", "bot"), ("tool", "bot"), ("query", "bot")]
|
||
retrieval = entries[0]
|
||
[retrieve_end] = await events(row.id, "retrieve.end", "docs")
|
||
assert retrieval == {"kind": "retrieval", "node_id": "docs", "exec": 1, "artifact": retrieve_end.data["artifact"],
|
||
"source": "docs", "rows": 1}
|
||
ends = await events(row.id, "tool.end", "bot")
|
||
queries = [e for e in entries if e["kind"] == "query"]
|
||
assert [q["artifact"] for q in queries] == [e.data["query_artifact"] for e in ends if e.data.get("query_artifact")]
|
||
assert [q["via"] for q in queries] == [ends[0].data["artifact"], ends[2].data["artifact"]]
|
||
assert queries[0]["call_id"] == "call_0" and queries[1]["call_id"] == "call_2"
|
||
assert queries[1]["columns"] == ["refunds"] and queries[1]["rows"] == 1
|
||
|
||
# 工具调用记录带上编号和工件
|
||
bot = (await events(row.id, "node.finished", "bot"))[0].data
|
||
assert [e["kind"] for e in bot["evidence"]] == ["tool", "query", "tool", "tool", "query"]
|
||
transcript = artifact_store.load(bot["artifact"])["tool_calls"]
|
||
assert [t.get("alias") for t in transcript] == ["Q1", None, "Q2"]
|
||
assert transcript[0]["artifact"] == queries[0]["artifact"] and transcript[0]["via"] == queries[0]["via"]
|
||
assert transcript[0]["call_id"] == "call_0"
|
||
|
||
catalog = build_catalog(nodes={}, ledger=entries)
|
||
assert catalog["Q1"]["artifact"] == queries[0]["artifact"] and catalog["Q2"]["artifact"] == queries[1]["artifact"]
|
||
assert catalog["K1"]["artifact"] == retrieval["artifact"] and "1 条" in catalog["K1"]["label"]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 问数据那条路：input → agent → report → output，报告直接引用 agent 查过的格
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_report_cites_cells_the_agent_queried(engine_up, shop, monkeypatch):
|
||
from app.engine.evidence import iter_segments
|
||
from app.providers import mock_model
|
||
|
||
answer = "本周销售额 [[v:Q1.r0.gmv]]，共 [[v:Q1.r0.orders]] 单。[[see:Q1]]\n\n[[table:Q1 cols=gmv,orders]]"
|
||
|
||
def decide(self, messages):
|
||
if self.tools:
|
||
if not any(isinstance(m, ToolMessage) for m in messages):
|
||
return AIMessage(content="查。", tool_calls=[{"name": TOOL, "args": {"sql": SQL}, "id": "call_0"}])
|
||
return AIMessage(content="查完了。")
|
||
return AIMessage(content=answer)
|
||
|
||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
|
||
g = chain(
|
||
node("start", "input"),
|
||
node("bot", "agent", prompt="本周卖了多少", tools=[TOOL], approval="never"),
|
||
node("write", "report", instructions="回答用户的问题"),
|
||
node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}]),
|
||
)
|
||
row = await finish(g)
|
||
assert row.status == "succeeded", row.error
|
||
assert row.output["答案"] == ("本周销售额 600.5，共 3 单。\n\n| gmv | orders |\n| --- | --- |\n| 600.5 | 3 |")
|
||
assert row.output["_evidence"]["fields"] == ["答案"]
|
||
|
||
[checked] = await events(row.id, "report.checked", "write")
|
||
assert checked.data["ok"] is True and checked.data["violations"] == []
|
||
doc = artifact_store.load(checked.data["doc_artifact"])
|
||
query = next(e for e in await ledger(row.id) if e["kind"] == "query")
|
||
cited = [s for s in iter_segments(doc) if s.get("ref") == "v:Q1.r0.gmv"]
|
||
assert len(cited) == 2 # 正文一处、表里一处，指向同一格
|
||
assert {s["cite"]["eid"] for s in cited} == {make_eid("cell", query["artifact"], {"row": 0, "column": "gmv"})}
|
||
assert doc["catalog"]["Q1"]["artifact"] == query["artifact"] and doc["catalog"]["Q1"]["node_id"] == "bot"
|