mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:21:58 +00:00
移除仓库中的商业数据:删掉 21 个相关用例,其余标识符脱敏
这个仓库是公开的,而历史里带着一份客户 BI 仓的逐字转储:ANALYTICS(原 schema 名)下 53 张视图名、实测行数、字段结构表,外加一句写明取数方是谁 的注脚。还有一处把真人姓名和工作单位写进了检查脚本的断言里。这些都不是 化名,是能直接拿去对库的东西。 按"含敏感词的测试直接删"处理,共 21 个用例: test_agent_tool_loop.py 整个文件(2 个,agent 工具循环的真图验收) test_conversations.py 9 个(会话 CRUD、历史文本、上下文进 prompt) test_datasource.py 5 个(探查失败的实情、候选 schema 提示) test_copilot_context.py 3 个(数据源摘要的限定名) test_review.py 1 个(干净运行无信号) test_tool_args.py 1 个(call_tool 端到端) 辅助函数和产品源码不在"测试"之列,改为脱敏——否则删一个 _seeded 会连带 带走 8 个没被点名的用例。标识符一律换成中性名:shop→shop、 ANALYTICS→ANALYTICS、prod_shop→prod_shop、真人姓名→张三。 markdown 样本里那两条不是改标识符能解决的:行数、字段结构、业务域描述 整段都是客户的。换成同构的合成内容——领域换了,markdown 构造逐项对齐 (标题层级、有序/无序列表、三列表、带 inline code 的单元格、引用块), 渲染器的覆盖面没有变窄。check-stream 里那句"用的是真实输出"跟着改成实话。 286 → 265 passed,差值正好是删掉的 21 个。tsc 干净,decode / stream / ui 三个 check 不回归。 历史里的同一批内容另行用 git filter-repo 处理。
This commit is contained in:
@@ -1,180 +0,0 @@
|
|||||||
"""Agent 的工具循环,跑真图验收。
|
|
||||||
|
|
||||||
单测能保证 split_tool_calls 分得对、prepare_args 纠得对,但保证不了它们在
|
|
||||||
run_agent 里被接对了。这里从 run_manager.start 进去、从落库的 RunEvent 出来,
|
|
||||||
验的是用户真正会看到的那条事件流。
|
|
||||||
|
|
||||||
两件必须守住的事:
|
|
||||||
|
|
||||||
- 每个 tool_use 都要有配对的 tool_result。少一条,下一次模型调用直接 400。
|
|
||||||
- 没执行的调用绝不能发 tool.start。前端靠 call_id 配对 start/end
|
|
||||||
(frontend/src/run/decode.ts),一条没有结尾的 start 就是界面上一个永远转圈的卡片。
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import sqlite3
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from langchain_core.messages import AIMessage, ToolMessage
|
|
||||||
from sqlalchemy import select
|
|
||||||
|
|
||||||
from app.db.base import SessionLocal
|
|
||||||
from app.db.models import DataSource, Run, RunEvent
|
|
||||||
from app.engine.runner import run_manager
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
async def runner():
|
|
||||||
"""真图要跑起来就得有 checkpointer —— 平时是 FastAPI 的 lifespan 建的。"""
|
|
||||||
await run_manager.setup()
|
|
||||||
yield run_manager
|
|
||||||
await run_manager.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
async def shop(tmp_path):
|
|
||||||
"""一个真实的小库 + 一条 DataSource 记录,工具要能真的查到数。"""
|
|
||||||
path = tmp_path / "shop.db"
|
|
||||||
db = sqlite3.connect(path)
|
|
||||||
db.executescript("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount REAL NOT NULL);")
|
|
||||||
db.executemany("INSERT INTO orders(id,amount) VALUES(?,?)", [(i, i * 2.0) for i in range(1, 8)])
|
|
||||||
db.commit()
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
name = f"shop_{tmp_path.name[-6:].replace('-', '_')}"
|
|
||||||
async with SessionLocal() as session:
|
|
||||||
session.add(DataSource(
|
|
||||||
name=name, kind="sqlite", database=str(path), readonly=True,
|
|
||||||
description="测试库", options={}, schema_cache={}, enabled=True,
|
|
||||||
))
|
|
||||||
await session.commit()
|
|
||||||
return name, str(path)
|
|
||||||
|
|
||||||
|
|
||||||
def _graph(tool_name: str) -> dict:
|
|
||||||
return {
|
|
||||||
"nodes": [
|
|
||||||
{"id": "in", "type": "input", "data": {"config": {}}},
|
|
||||||
{"id": "bot", "type": "agent", "data": {"config": {
|
|
||||||
"prompt": "查一下订单总额",
|
|
||||||
"tools": [tool_name],
|
|
||||||
"approval": "never", # 审批会挂起,这里要一路跑到底
|
|
||||||
"max_steps": 3,
|
|
||||||
}}},
|
|
||||||
{"id": "out", "type": "output", "data": {"config": {}}},
|
|
||||||
],
|
|
||||||
"edges": [
|
|
||||||
{"source": "in", "target": "bot"},
|
|
||||||
{"source": "bot", "target": "out"},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def _run_to_completion(graph: dict) -> list[RunEvent]:
|
|
||||||
run = await run_manager.start(graph=graph, input_payload={"question": "订单总额"})
|
|
||||||
for _ in range(300):
|
|
||||||
await asyncio.sleep(0.05)
|
|
||||||
async with SessionLocal() as session:
|
|
||||||
row = (await session.execute(select(Run).where(Run.id == run.id))).scalar_one()
|
|
||||||
if row.status in ("succeeded", "failed", "cancelled"):
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
pytest.fail("运行没有在 15 秒内结束")
|
|
||||||
|
|
||||||
async with SessionLocal() as session:
|
|
||||||
events = list((await session.execute(
|
|
||||||
select(RunEvent).where(RunEvent.run_id == run.id).order_by(RunEvent.seq)
|
|
||||||
)).scalars())
|
|
||||||
return events, row
|
|
||||||
|
|
||||||
|
|
||||||
async def test_serial_loop_runs_one_tool_and_answers_every_call(runner, shop, monkeypatch) -> None:
|
|
||||||
"""模型一轮发两个工具调用:只跑第一个,但两个都要有回音。
|
|
||||||
|
|
||||||
顺带把第一个的参数名写错成 query —— 这正是那条
|
|
||||||
"_run() got an unexpected keyword argument 'query'" 的来源。
|
|
||||||
"""
|
|
||||||
name, _ = shop
|
|
||||||
tool_name = f"db_query__{name}"
|
|
||||||
|
|
||||||
from app.providers import mock_model
|
|
||||||
|
|
||||||
turns = {"n": 0}
|
|
||||||
|
|
||||||
def _decide(self, messages):
|
|
||||||
turns["n"] += 1
|
|
||||||
if turns["n"] == 1:
|
|
||||||
return AIMessage(
|
|
||||||
content="先查一下。",
|
|
||||||
tool_calls=[
|
|
||||||
# 参数名故意写错:能被纠正成 sql 才算过
|
|
||||||
{"name": tool_name, "args": {"query": "SELECT SUM(amount) AS total FROM orders"},
|
|
||||||
"id": "call_a"},
|
|
||||||
{"name": tool_name, "args": {"sql": "SELECT COUNT(*) FROM orders"},
|
|
||||||
"id": "call_b"},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
return AIMessage(content="订单总额是 56。")
|
|
||||||
|
|
||||||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
|
||||||
|
|
||||||
events, run = await _run_to_completion(_graph(tool_name))
|
|
||||||
kinds = [(e.type, e.data) for e in events]
|
|
||||||
|
|
||||||
assert run.status == "succeeded", [k for k, _ in kinds]
|
|
||||||
|
|
||||||
# 一轮只跑了一个工具
|
|
||||||
starts = [d for t, d in kinds if t == "tool.start"]
|
|
||||||
assert len(starts) == 1, f"应该只有一次 tool.start,实际 {len(starts)} 次"
|
|
||||||
assert starts[0]["call_id"] == "call_a"
|
|
||||||
# 发出去的是纠正后的参数,不是模型原样那份
|
|
||||||
assert "sql" in starts[0]["args"] and "query" not in starts[0]["args"]
|
|
||||||
|
|
||||||
# 每个 tool.start 都有配对的结尾,界面上不会留下转圈的卡片
|
|
||||||
ends = [d for t, d in kinds if t in ("tool.end", "tool.error")]
|
|
||||||
assert {d["call_id"] for d in ends} == {"call_a"}
|
|
||||||
|
|
||||||
# 纠正这件事说出来了,没有安静地把事办了
|
|
||||||
warns = [d["message"] for t, d in kinds if t == "log" and d.get("level") == "warn"]
|
|
||||||
assert any("query" in w and "sql" in w for w in warns), warns
|
|
||||||
|
|
||||||
# 工具真的查到了数
|
|
||||||
assert "56" in str([d.get("preview") for t, d in kinds if t == "tool.end"])
|
|
||||||
|
|
||||||
|
|
||||||
async def test_bad_args_feed_the_schema_back_instead_of_dying(runner, shop, monkeypatch) -> None:
|
|
||||||
"""参数全不认识时不执行,把"它接受什么"喂回去——模型下一步才改得对。"""
|
|
||||||
name, _ = shop
|
|
||||||
tool_name = f"db_query__{name}"
|
|
||||||
|
|
||||||
from app.providers import mock_model
|
|
||||||
|
|
||||||
seen: list[str] = []
|
|
||||||
turns = {"n": 0}
|
|
||||||
|
|
||||||
def _decide(self, messages):
|
|
||||||
turns["n"] += 1
|
|
||||||
if turns["n"] == 1:
|
|
||||||
return AIMessage(
|
|
||||||
content="查一下。",
|
|
||||||
tool_calls=[{"name": tool_name, "args": {"foo": 1, "bar": 2}, "id": "call_x"}],
|
|
||||||
)
|
|
||||||
# 第二轮:把上一轮喂回来的内容记下来,这就是模型能看到的全部依据
|
|
||||||
seen.extend(m.content for m in messages if isinstance(m, ToolMessage))
|
|
||||||
return AIMessage(content="好的。")
|
|
||||||
|
|
||||||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
|
||||||
|
|
||||||
events, run = await _run_to_completion(_graph(tool_name))
|
|
||||||
assert run.status == "succeeded"
|
|
||||||
|
|
||||||
assert seen, "参数错了也必须给模型一条 tool_result,否则下一轮直接 400"
|
|
||||||
text = seen[0]
|
|
||||||
assert "sql" in text and "必填" in text # 说清楚该填什么
|
|
||||||
assert "unexpected keyword argument" not in text # 不再是那句天书
|
|
||||||
|
|
||||||
# 没执行的调用不发 tool.start,界面上不会留下转不完的卡片
|
|
||||||
assert not [e for e in events if e.type == "tool.start"]
|
|
||||||
assert [e for e in events if e.type == "tool.error"]
|
|
||||||
@@ -51,41 +51,6 @@ async def _turn(client, conv_id: str, question: str, **patch) -> dict:
|
|||||||
# --------------------------------------------------------------------------
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
async def test_a_conversation_survives_and_lists(client) -> None:
|
|
||||||
conv = await _conversation(client)
|
|
||||||
await _turn(client, conv["id"], "shop 里有几家店", answer="12 家", status="done")
|
|
||||||
|
|
||||||
# 列表里认得出是哪次聊天
|
|
||||||
rows = (await client.get("/api/conversations")).json()
|
|
||||||
mine = next(r for r in rows if r["id"] == conv["id"])
|
|
||||||
assert mine["turn_count"] == 1
|
|
||||||
assert mine["last_question"] == "shop 里有几家店"
|
|
||||||
|
|
||||||
# 完整取回:这是刷新页面后重建界面的唯一依据
|
|
||||||
detail = (await client.get(f"/api/conversations/{conv['id']}")).json()
|
|
||||||
assert [t["question"] for t in detail["turns"]] == ["shop 里有几家店"]
|
|
||||||
assert detail["turns"][0]["answer"] == "12 家"
|
|
||||||
|
|
||||||
|
|
||||||
async def test_title_comes_from_the_first_question(client) -> None:
|
|
||||||
"""一排"新对话"等于没有列表。"""
|
|
||||||
conv = await _conversation(client)
|
|
||||||
assert conv["title"] == "新对话"
|
|
||||||
await _turn(client, conv["id"], "shop 里有几家店")
|
|
||||||
assert (await client.get(f"/api/conversations/{conv['id']}")).json()["title"] == "shop 里有几家店"
|
|
||||||
|
|
||||||
# 第二轮不该再改标题
|
|
||||||
await _turn(client, conv["id"], "再按月份拆一下")
|
|
||||||
assert (await client.get(f"/api/conversations/{conv['id']}")).json()["title"] == "shop 里有几家店"
|
|
||||||
|
|
||||||
|
|
||||||
async def test_a_renamed_conversation_keeps_its_name(client) -> None:
|
|
||||||
conv = await _conversation(client)
|
|
||||||
await client.patch(f"/api/conversations/{conv['id']}", json={"title": "门店盘点"})
|
|
||||||
await _turn(client, conv["id"], "shop 里有几家店")
|
|
||||||
assert (await client.get(f"/api/conversations/{conv['id']}")).json()["title"] == "门店盘点"
|
|
||||||
|
|
||||||
|
|
||||||
def test_long_titles_are_cut_not_wrapped() -> None:
|
def test_long_titles_are_cut_not_wrapped() -> None:
|
||||||
assert title_from("短问题") == "短问题"
|
assert title_from("短问题") == "短问题"
|
||||||
assert title_from("") == "新对话"
|
assert title_from("") == "新对话"
|
||||||
@@ -150,33 +115,6 @@ async def _seeded(client) -> str:
|
|||||||
return conv["id"]
|
return conv["id"]
|
||||||
|
|
||||||
|
|
||||||
async def test_history_text_is_plain_qa(client) -> None:
|
|
||||||
conv_id = await _seeded(client)
|
|
||||||
async with SessionLocal() as session:
|
|
||||||
text = await history_text(session, conv_id)
|
|
||||||
assert "shop 里有几家店" in text and "一共 12 家" in text
|
|
||||||
|
|
||||||
|
|
||||||
async def test_history_carries_the_review_warning_forward(client) -> None:
|
|
||||||
"""被复核判过不可信的答案,下一轮必须知道。
|
|
||||||
|
|
||||||
只把正文喂过去的话,模型会拿一个已知有问题的结论当事实继续往下推
|
|
||||||
("那 12 家里有几家是新开的?"),错误就这么一轮轮传下去,而且越传
|
|
||||||
越像真的——后面几轮看起来都在正常作答。
|
|
||||||
"""
|
|
||||||
conv = await _conversation(client)
|
|
||||||
await _turn(
|
|
||||||
client, conv["id"], "shop 里有几家店",
|
|
||||||
answer="一共 12 家", status="done",
|
|
||||||
review={"verdict": "annotated", "severity": "broken", "note": "取数没跑通,这个数不可信。",
|
|
||||||
"answer": None, "retry": True, "signals": []},
|
|
||||||
)
|
|
||||||
async with SessionLocal() as session:
|
|
||||||
text = await history_text(session, conv["id"])
|
|
||||||
assert "一共 12 家" in text
|
|
||||||
assert "取数没跑通" in text
|
|
||||||
|
|
||||||
|
|
||||||
async def test_history_text_is_empty_without_a_conversation(client) -> None:
|
async def test_history_text_is_empty_without_a_conversation(client) -> None:
|
||||||
async with SessionLocal() as session:
|
async with SessionLocal() as session:
|
||||||
assert await history_text(session, None) == ""
|
assert await history_text(session, None) == ""
|
||||||
@@ -212,61 +150,11 @@ async def test_a_huge_answer_does_not_crowd_out_the_other_turns(client) -> None:
|
|||||||
assert "那总共多少" in text # 后一轮没被前一轮挤掉
|
assert "那总共多少" in text # 后一轮没被前一轮挤掉
|
||||||
|
|
||||||
|
|
||||||
async def test_history_section_carries_the_previous_graph(client) -> None:
|
|
||||||
"""只给问答文字不够:追问会让模型重新设计一张图,可能接到别的表上。"""
|
|
||||||
from app.api.copilot import _history_section
|
|
||||||
|
|
||||||
conv_id = await _seeded(client)
|
|
||||||
async with SessionLocal() as session:
|
|
||||||
block = await _history_section(session, conv_id)
|
|
||||||
|
|
||||||
assert "shop 里有几家店" in block
|
|
||||||
assert "shop_agent" in block and "db_query__shop" in block
|
|
||||||
assert '"x"' not in block and '"position"' not in block # _slim 去掉了坐标噪音
|
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------
|
# --------------------------------------------------------------------------
|
||||||
# 上下文真的到了模型手上
|
# 上下文真的到了模型手上
|
||||||
# --------------------------------------------------------------------------
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
async def test_the_previous_turn_reaches_the_prompt(client, monkeypatch) -> None:
|
|
||||||
"""整件事的关键一环。
|
|
||||||
|
|
||||||
拼字符串的函数测通过、却没接进 messages,是这类功能最典型的失败方式:
|
|
||||||
界面上完全看不出来,它只是"答得不太对"。所以这里拦在模型入口上,
|
|
||||||
看送出去的那条 human message 里到底有没有上一轮。
|
|
||||||
"""
|
|
||||||
import app.api.copilot as copilot
|
|
||||||
|
|
||||||
conv_id = await _seeded(client)
|
|
||||||
captured: list = []
|
|
||||||
|
|
||||||
class _Recorder:
|
|
||||||
def with_structured_output(self, *_a, **_k):
|
|
||||||
return self
|
|
||||||
|
|
||||||
async def ainvoke(self, messages):
|
|
||||||
captured.append(messages)
|
|
||||||
return {"nodes": [], "edges": []}
|
|
||||||
|
|
||||||
async def _model(*_a, **_k):
|
|
||||||
return _Recorder(), "mock-fast"
|
|
||||||
|
|
||||||
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
|
||||||
|
|
||||||
r = await client.post("/api/copilot/generate", json={
|
|
||||||
"instruction": "再按月份拆一下", "intent": "answer", "conversation_id": conv_id,
|
|
||||||
})
|
|
||||||
assert r.status_code == 200, r.text
|
|
||||||
|
|
||||||
human = next(text for role, text in captured[0] if role == "human")
|
|
||||||
assert "shop 里有几家店" in human # 上一轮问的
|
|
||||||
assert "一共 12 家" in human # 上一轮答的
|
|
||||||
assert "shop_agent" in human # 上一轮的图
|
|
||||||
assert "再按月份拆一下" in human # 这一轮问的
|
|
||||||
|
|
||||||
|
|
||||||
async def test_no_conversation_means_no_history_header(client, monkeypatch) -> None:
|
async def test_no_conversation_means_no_history_header(client, monkeypatch) -> None:
|
||||||
"""不带 conversation_id 时不能凭空多出一个空的历史标题。"""
|
"""不带 conversation_id 时不能凭空多出一个空的历史标题。"""
|
||||||
import app.api.copilot as copilot
|
import app.api.copilot as copilot
|
||||||
@@ -458,44 +346,11 @@ def test_the_three_paths_are_spelled_out_for_the_model() -> None:
|
|||||||
assert '"run":true' in _STREAM_PROTOCOL
|
assert '"run":true' in _STREAM_PROTOCOL
|
||||||
|
|
||||||
|
|
||||||
async def test_copilot_is_told_not_to_invent_a_memory_scope() -> None:
|
|
||||||
"""记忆域由平台给。图里自己起一个,写进去就再也读不回来。
|
|
||||||
|
|
||||||
真发生过(会话 cbe09f4b 的第 2~3 轮):一轮的 memory 节点写了
|
|
||||||
scope="user",下一轮没写 scope、落回 default,于是「我叫张三」存下来了,
|
|
||||||
问「我是谁」却答不上来——记忆在库里好好的,只是读错了地方。
|
|
||||||
"""
|
|
||||||
from app.api.copilot import NODE_REFERENCE
|
|
||||||
|
|
||||||
memory_line = next(
|
|
||||||
(l for l in NODE_REFERENCE.splitlines() if l.strip().startswith("- memory:")), "")
|
|
||||||
assert memory_line, "节点参考里找不到 memory 这一行"
|
|
||||||
assert "scope" not in memory_line, "scope 还列在可填字段里,模型就会去填它"
|
|
||||||
assert "不要写 scope" in NODE_REFERENCE
|
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------
|
# --------------------------------------------------------------------------
|
||||||
# 主动记忆
|
# 主动记忆
|
||||||
# --------------------------------------------------------------------------
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def test_copilot_is_told_when_to_record_a_memory() -> None:
|
|
||||||
"""以前只有用户明说「记住」才会写:17 轮里只有 4 轮带记忆节点,全在用户
|
|
||||||
主动谈记忆的那个会话里。于是他说过一次「我叫张三」,换个话题再问就
|
|
||||||
谁也不知道了。
|
|
||||||
|
|
||||||
但边界比指令本身更要紧——记错了会污染以后每一次对话。
|
|
||||||
"""
|
|
||||||
from app.api.copilot import GenerateIn, _user_message
|
|
||||||
|
|
||||||
text = _user_message(GenerateIn(instruction="随便问问", intent="answer"), patch=False)
|
|
||||||
assert "该记" in text and "不该记" in text
|
|
||||||
# 最容易被记进去的垃圾:这一轮的问题、查出来的数、一次性指令、以及推测
|
|
||||||
for junk in ("这一轮的问题本身", "查出来的数", "一次性的指令", "推测出来的东西"):
|
|
||||||
assert junk in text, f"没有拦住「{junk}」这类"
|
|
||||||
assert "拿不准就不记" in text
|
|
||||||
|
|
||||||
|
|
||||||
async def test_writing_a_memory_is_visible_in_the_trace(client, monkeypatch) -> None:
|
async def test_writing_a_memory_is_visible_in_the_trace(client, monkeypatch) -> None:
|
||||||
"""自动记忆的前提是看得见。
|
"""自动记忆的前提是看得见。
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from app.api.copilot import GRAPH_SCHEMA, _DATASOURCE_BUDGET_CHARS
|
|||||||
from app.data.introspect import summary
|
from app.data.introspect import summary
|
||||||
|
|
||||||
|
|
||||||
def _source(n_tables: int, *, name: str = "bi", schema: str | None = "ANALYTICS"):
|
def _source(n_tables: int, *, name: str = "warehouse", schema: str | None = "ANALYTICS"):
|
||||||
tables = {
|
tables = {
|
||||||
f"t{i}": {
|
f"t{i}": {
|
||||||
"qualified": f"{schema}.t{i}" if schema else f"t{i}",
|
"qualified": f"{schema}.t{i}" if schema else f"t{i}",
|
||||||
@@ -45,19 +45,6 @@ def test_graph_schema_allows_arbitrary_config_keys():
|
|||||||
assert "config" in node["required"]
|
assert "config" in node["required"]
|
||||||
|
|
||||||
|
|
||||||
def test_summary_detail_mode_lists_columns():
|
|
||||||
text = summary(_source(3))
|
|
||||||
assert "ANALYTICS.t0" in text and "c0" in text
|
|
||||||
|
|
||||||
|
|
||||||
def test_summary_compact_mode_drops_columns():
|
|
||||||
"""库多了要靠它压 token:只留对象名,字段让 db_schema 按需查。"""
|
|
||||||
text = summary(_source(50), detail=False)
|
|
||||||
assert "ANALYTICS.t0" in text
|
|
||||||
assert "c0" not in text
|
|
||||||
assert len(text) < len(summary(_source(50)))
|
|
||||||
|
|
||||||
|
|
||||||
def test_compact_mode_is_substantially_smaller():
|
def test_compact_mode_is_substantially_smaller():
|
||||||
"""压缩比要真的有效,否则降级没意义。实测约 35%。"""
|
"""压缩比要真的有效,否则降级没意义。实测约 35%。"""
|
||||||
full = summary(_source(53))
|
full = summary(_source(53))
|
||||||
@@ -65,11 +52,6 @@ def test_compact_mode_is_substantially_smaller():
|
|||||||
assert len(compact) < len(full) * 0.5
|
assert len(compact) < len(full) * 0.5
|
||||||
|
|
||||||
|
|
||||||
def test_qualified_names_appear_in_summary():
|
|
||||||
"""摘要里必须是带 schema 的全名——模型照抄,漏前缀就是 ORA-00942。"""
|
|
||||||
assert "ANALYTICS.t0" in summary(_source(2))
|
|
||||||
|
|
||||||
|
|
||||||
def test_summary_without_schema_uses_bare_names():
|
def test_summary_without_schema_uses_bare_names():
|
||||||
"""没有 schema 概念的库不要硬拼前缀,否则生成的 SQL 反而错。"""
|
"""没有 schema 概念的库不要硬拼前缀,否则生成的 SQL 反而错。"""
|
||||||
text = summary(_source(2, schema=None))
|
text = summary(_source(2, schema=None))
|
||||||
|
|||||||
@@ -197,32 +197,6 @@ async def test_qualified_name_includes_schema_when_given(source):
|
|||||||
assert cache["tables"]["orders"]["qualified"] == "main.orders"
|
assert cache["tables"]["orders"]["qualified"] == "main.orders"
|
||||||
|
|
||||||
|
|
||||||
async def test_bad_schema_returns_candidates_instead_of_raising(source, monkeypatch):
|
|
||||||
"""schema 填错是高频事故(大小写、拼写),要给可选项而不是抛驱动异常。"""
|
|
||||||
from app.data import introspect as mod
|
|
||||||
|
|
||||||
async def fake_list(_source):
|
|
||||||
return ["ANALYTICS", "STAGING"]
|
|
||||||
|
|
||||||
monkeypatch.setattr(mod, "list_schemas", fake_list)
|
|
||||||
cache = await mod.introspect(source, schema="根本不存在")
|
|
||||||
assert cache["tables"] == {}
|
|
||||||
assert cache["available_schemas"] == ["ANALYTICS", "STAGING"]
|
|
||||||
|
|
||||||
|
|
||||||
async def test_summary_points_at_candidate_schemas(source, monkeypatch):
|
|
||||||
"""摘要要把候选说给 Copilot 听,否则它只会对着空结构编表名。"""
|
|
||||||
from app.data import introspect as mod
|
|
||||||
|
|
||||||
async def fake_list(_source):
|
|
||||||
return ["ANALYTICS", "STAGING"]
|
|
||||||
|
|
||||||
monkeypatch.setattr(mod, "list_schemas", fake_list)
|
|
||||||
source.schema_cache = await mod.introspect(source, schema="根本不存在")
|
|
||||||
text = summary(source)
|
|
||||||
assert "ANALYTICS" in text and "该账号名下没有对象" in text
|
|
||||||
|
|
||||||
|
|
||||||
def test_schema_stats_sql_covers_every_supported_kind():
|
def test_schema_stats_sql_covers_every_supported_kind():
|
||||||
"""新增数据库类型时别忘了这里——漏了就退化成黑名单猜测。"""
|
"""新增数据库类型时别忘了这里——漏了就退化成黑名单猜测。"""
|
||||||
from app.data.engine import SUPPORTED_KINDS
|
from app.data.engine import SUPPORTED_KINDS
|
||||||
@@ -277,60 +251,3 @@ def test_why_empty_tells_the_two_cases_apart():
|
|||||||
assert "失败" in why_empty(_FailedCache.schema_cache)
|
assert "失败" in why_empty(_FailedCache.schema_cache)
|
||||||
assert "Lost connection" in why_empty(_FailedCache.schema_cache)
|
assert "Lost connection" in why_empty(_FailedCache.schema_cache)
|
||||||
assert why_empty({}) == "还没有探查过结构"
|
assert why_empty({}) == "还没有探查过结构"
|
||||||
|
|
||||||
|
|
||||||
def test_schema_tool_tells_the_agent_the_truth():
|
|
||||||
"""给 agent 的那句话要是实情,而且要给一个它做得到的下一步。"""
|
|
||||||
from app.tools.datasource import _no_schema_note
|
|
||||||
|
|
||||||
note = _no_schema_note(_FailedCache)
|
|
||||||
assert "失败" in note and "Lost connection" in note
|
|
||||||
# 候选一直在缓存里,以前没人交出去
|
|
||||||
assert "prod_shop" in note and "toolsdb" in note
|
|
||||||
# agent 进不了设置页。它做得到的是自己查数据字典——那条路它最后也走通了,
|
|
||||||
# 只是自己摸索了三轮
|
|
||||||
assert "information_schema" in note
|
|
||||||
assert "不要猜表名" in note
|
|
||||||
# 不能再说"请到设置页"
|
|
||||||
assert "设置页" not in note
|
|
||||||
|
|
||||||
fresh = _no_schema_note(_NeverRun)
|
|
||||||
assert "还没有探查过" in fresh
|
|
||||||
assert "失败" not in fresh
|
|
||||||
|
|
||||||
|
|
||||||
def test_describe_table_does_not_claim_the_table_is_missing():
|
|
||||||
"""缓存是空的时候,说"里面没有这张表"是在误导——真相是我们什么都不知道。"""
|
|
||||||
from app.data.introspect import describe_table
|
|
||||||
|
|
||||||
out = describe_table(_FailedCache, "user")
|
|
||||||
assert "无法判断" in out
|
|
||||||
assert "失败" in out
|
|
||||||
# 以前是 "数据源「shop」里没有 user。现有的对象:(还没探查过结构)"
|
|
||||||
assert "现有的对象" not in out
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_introspect_failure_does_not_get_a_synced_stamp(client, monkeypatch):
|
|
||||||
"""失败要把缓存留下(候选是下一步的线索),但不能盖"已同步"的戳。
|
|
||||||
|
|
||||||
以前盖了,于是界面说同步过、工具说还没探查过,两边都不说真话。
|
|
||||||
"""
|
|
||||||
import app.api.datasources as mod
|
|
||||||
|
|
||||||
async def failed(source, schema=None):
|
|
||||||
return dict(_FailedCache.schema_cache)
|
|
||||||
|
|
||||||
created = await client.post("/api/datasources", json={
|
|
||||||
"name": "introspect_fail", "kind": "sqlite", "database": ":memory:",
|
|
||||||
})
|
|
||||||
assert created.status_code == 201, created.text
|
|
||||||
sid = created.json()["id"]
|
|
||||||
|
|
||||||
monkeypatch.setattr(mod.introspect_mod, "introspect", failed)
|
|
||||||
out = (await client.post(f"/api/datasources/{sid}/introspect")).json()
|
|
||||||
|
|
||||||
assert out["schema_synced_at"] is None, "失败不该盖已同步的戳"
|
|
||||||
assert "Lost connection" in out["schema_error"]
|
|
||||||
assert out["available_schemas"] == ["prod_shop", "shop_dev", "toolsdb"]
|
|
||||||
assert out["table_count"] == 0
|
|
||||||
|
|||||||
@@ -39,19 +39,6 @@ OK_OUTPUT = {"结果": "管理员有 6 个。"}
|
|||||||
# --------------------------------------------------------------------------
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def test_clean_run_has_no_signals():
|
|
||||||
"""干净的运行扫不出东西——这是"零成本"的前提。"""
|
|
||||||
events = [
|
|
||||||
ev(EventType.RUN_STARTED),
|
|
||||||
ev(EventType.TOOL_START, tool="db_query__shop"),
|
|
||||||
ev(EventType.TOOL_END, tool="db_query__shop"),
|
|
||||||
ev(EventType.LOG, level="info", message="随便什么"),
|
|
||||||
ev(EventType.RETRIEVE_END, count=5, degraded=False, query="权限"),
|
|
||||||
ev(EventType.RUN_FINISHED),
|
|
||||||
]
|
|
||||||
assert rv.scan(events, OK_OUTPUT) == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_tool_error_is_broken():
|
def test_tool_error_is_broken():
|
||||||
signals = rv.scan([ev(EventType.TOOL_ERROR, tool="db_query__x", error="no such table")],
|
signals = rv.scan([ev(EventType.TOOL_ERROR, tool="db_query__x", error="no such table")],
|
||||||
OK_OUTPUT)
|
OK_OUTPUT)
|
||||||
|
|||||||
@@ -172,40 +172,6 @@ async def test_schema_tool_args_are_validated_too(source) -> None:
|
|||||||
assert payload["table"] == "orders" and note is not None
|
assert payload["table"] == "orders" and note is not None
|
||||||
|
|
||||||
|
|
||||||
async def test_call_tool_end_to_end_on_the_path_that_broke(source) -> None:
|
|
||||||
"""走 call_tool 这个真实入口(工具节点和设置页的"试一下"都走它)。
|
|
||||||
|
|
||||||
以前这条路径对动态工具是裸 `**args` 展开的,报错止步于 TypeError;
|
|
||||||
现在参数名能被纠正、纠正这件事能通过 on_fix 说出去。
|
|
||||||
"""
|
|
||||||
from app.db.base import SessionLocal
|
|
||||||
from app.db.models import DataSource
|
|
||||||
from app.tools.registry import call_tool
|
|
||||||
|
|
||||||
async with SessionLocal() as session:
|
|
||||||
session.add(DataSource(
|
|
||||||
name="shop", kind="sqlite", database=source.database,
|
|
||||||
readonly=True, description="测试库", options={}, schema_cache={}, enabled=True,
|
|
||||||
))
|
|
||||||
await session.commit()
|
|
||||||
|
|
||||||
notes: list[str] = []
|
|
||||||
async with SessionLocal() as session:
|
|
||||||
out = await call_tool(
|
|
||||||
"db_query__shop", {"query": "SELECT COUNT(*) AS c FROM orders"},
|
|
||||||
ToolContext(), session=session, on_fix=notes.append,
|
|
||||||
)
|
|
||||||
assert "5" in str(out)
|
|
||||||
# 跑通了,但配置里那个错的参数名还在,必须留下痕迹而不是安静地把事办了
|
|
||||||
assert len(notes) == 1 and "sql" in notes[0]
|
|
||||||
|
|
||||||
async with SessionLocal() as session:
|
|
||||||
with pytest.raises(ToolArgsError) as e:
|
|
||||||
await call_tool("db_query__shop", {"foo": 1, "bar": 2}, ToolContext(), session=session)
|
|
||||||
assert "sql" in str(e.value)
|
|
||||||
assert "unexpected keyword argument" not in str(e.value)
|
|
||||||
|
|
||||||
|
|
||||||
def test_tools_without_a_pydantic_schema_are_let_through() -> None:
|
def test_tools_without_a_pydantic_schema_are_let_through() -> None:
|
||||||
"""MCP 这类工具的 args_schema 可能是一份 JSON Schema dict,校验不了就放行。"""
|
"""MCP 这类工具的 args_schema 可能是一份 JSON Schema dict,校验不了就放行。"""
|
||||||
assert args_model_of(SimpleNamespace(args_schema={"type": "object"})) is None
|
assert args_model_of(SimpleNamespace(args_schema={"type": "object"})) is None
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
// 用**真实运行**导出的事件跑(fixtures.json 取自 data/agentlab.db),不是构造的样本:
|
// 用**真实运行**导出的事件跑(fixtures.json 取自 data/agentlab.db),不是构造的样本:
|
||||||
// 这里要守的恰恰是真实事件的脏细节——审批会重放、成对事件字段不齐、
|
// 这里要守的恰恰是真实事件的脏细节——审批会重放、成对事件字段不齐、
|
||||||
// 空 preview、节点没起过名字。构造的样本永远不会长成那样。
|
// 空 preview、节点没起过名字。构造的样本永远不会长成那样。
|
||||||
|
// 库名表名已脱敏(改的只是标识符字面量,事件的结构和脏细节原样保留)。
|
||||||
//
|
//
|
||||||
// 转译借 vite dev server(它 serve 的就是应用实际运行的那份),所以跑之前
|
// 转译借 vite dev server(它 serve 的就是应用实际运行的那份),所以跑之前
|
||||||
// 前端得起着:./scripts/dev.sh
|
// 前端得起着:./scripts/dev.sh
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ console.log('\n=== 展开交互 ===')
|
|||||||
await page.waitForTimeout(300)
|
await page.waitForTimeout(300)
|
||||||
const before = await page.locator('body').innerText()
|
const before = await page.locator('body').innerText()
|
||||||
check('收起时不显示 SQL', !before.includes('SELECT'))
|
check('收起时不显示 SQL', !before.includes('SELECT'))
|
||||||
await page.locator('button', { hasText: '在 bi 上查询数据' }).first().click()
|
await page.locator('button', { hasText: '在 warehouse 上查询数据' }).first().click()
|
||||||
await page.waitForTimeout(250)
|
await page.waitForTimeout(250)
|
||||||
const after = await page.locator('body').innerText()
|
const after = await page.locator('body').innerText()
|
||||||
check('展开后能看到 SQL 原文', after.includes('SELECT') && after.includes('ANALYTICS'))
|
check('展开后能看到 SQL 原文', after.includes('SELECT') && after.includes('ANALYTICS'))
|
||||||
@@ -67,8 +67,10 @@ console.log('\n=== 展开交互 ===')
|
|||||||
|
|
||||||
console.log('\n=== Markdown 渲染 ===')
|
console.log('\n=== Markdown 渲染 ===')
|
||||||
{
|
{
|
||||||
// 用库里导出的**真实输出**,不是编的样本:模型写什么才是要渲染的,
|
// 样本取自库里导出的**真实输出**,不是按 CommonMark 规范挑的测例:模型
|
||||||
// 按 CommonMark 规范挑测例只会测到用不上的角落
|
// 实际会写成什么样,才是要渲染的东西。涉及业务数据的那几条已换成同构的
|
||||||
|
// 合成内容——换的是领域,markdown 构造(标题层级、表格、引用块、inline
|
||||||
|
// code 密度)逐项对齐,覆盖面没有变窄
|
||||||
const page = await browser.newPage({ viewport: { width: 1100, height: 900 } })
|
const page = await browser.newPage({ viewport: { width: 1100, height: 900 } })
|
||||||
const errors = []
|
const errors = []
|
||||||
page.on('pageerror', (e) => errors.push(e.message))
|
page.on('pageerror', (e) => errors.push(e.message))
|
||||||
|
|||||||
Reference in New Issue
Block a user