Files
AgentLab/backend/tests/test_conversations.py
T
ThearchyHelios cc608f56ec 跑完先过一道复核再交给你:异常说破、数字不许现编、出错自动改一次
「跑完了」和「答得对」之间一直没有任何一层。库里 264 条 succeeded 里,
14 条带着 warn、4 条 agent 步数用满,而交到用户手上的答案对此只字不提。
最糟的一次(run dd9927e6)交出去的干脆是「本轮只执行了第一个工具」——
一句内部管道文案冒充结论。

出具体系(issuance.py)本来就是干这个的,但它挂在输出节点的可选契约上,
还要配口径卡提供受控指标集;问数据页生成的图基本都是 input → agent → output,
走不到那条路。所以补一层轻的。

**干净的运行一次模型都不调。** 先用规则扫一遍事件流(engine/review.py:scan),
没扫出东西就到此为止。实测 15ms 返回。这是整层设计的前提,测试里用
「monkeypatch 一个会抛异常的模型,断言它没被碰过」来守。

**分类挂在 code 上,不是中文消息串。** 12 处 warn 发射点都带上 code。
这样「流式失败,回退非流式」能归成 benign 直接跳过——它对答案内容零影响,
为它叫一次模型等于每次网络抖一下就多花一笔;而「步数用满」归 broken。
原来这两者在事件流里长得一模一样。工具报错之后同一个工具又跑成了,
算 degraded 不算 broken——否则参数自愈每修一次就报一次「答案不可信」。

**复核不许现编数字。** 它手上只有原答案和一份异常清单,没有任何原始数据,
写出来的新数字只能是凭空来的。直接复用 issuance.trace_numbers,把原答案的
数字当受控指标集,对不上就整条退回原答案、只留说明。硬拦截,不是提示。

**规则管重试资格,模型只有否决权。** 一开始写成两边取并,实测 loop_limit
时模型判 retry=true——而重跑一遍循环还是会撞上限,白烧一次建图加一次跑图。
反过来之后,「检索什么都没查到」这类也不会被模型强推重跑。

**重试带着失败的那张图,不是推倒重来。** 原来传 base_graph=null,模型只看到
「原问题 + 一句失败原因」,很可能原样再建出同一张坏图。而且它连自己上次建了
什么都看不到——会话历史按 status != running 过滤,正在重试的这一轮恰好被
排除在外。现在交回去走后端的改图分支,调大步数、改正参数就是改一个字段。
上限一次。

**答案等复核完成再显示。** 端到端抓到的:答案先摆出来,十秒后说明才追上去——
这十秒里用户读的是一个没人核对过的结论,正是要修的那件事本身。改成落库立刻
(防关标签页丢结果)、显示等复核,并给复核加 25s 上限。说明排在成果上方,
broken 用报警色、degraded 用普通边框——混成一个样子用户就学会了一概忽略。

复核结论跟着轮次落库,刷新后还在;也进下一轮的对话历史,否则模型会拿一个
已知不可信的结论当事实继续往下推。

验证:后端 242 passed(新增 test_review.py 25 条);tsc 干净;
check-decode / check-stream / check-ui 全过,check-stream 新增 9 条渲染断言。
两条端到端都在真浏览器 + 真模型上跑通:检索降级那次说明正确排在答案上方;
重试那次建图请求 2 次、第二次 base_graph 带着失败的 3 个节点、第一版没漏出去。

复核结果不是事件流,decode.ts 翻译不到它,所以渲染测试放在 check-stream。
2026-09-20 11:18:53 +08:00

518 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""会话:持久化、列表,以及上下文真的进了 prompt。
起因是三条 run(d983c4e6 / 14103db5 / 08c3de1c)在用户看来是一次对话,
系统里却毫无关联:每轮 base_graph=None 从零建图,每轮一条独立的 thread_id,
而前端的 turns 只活在内存里——刷新页面就全没了。
这里最要紧的不是 CRUD,是最后那组:**上下文确实到了模型手上**。拼字符串的
函数单独测通过、但没接到 messages 里,是这类功能最典型的失败方式,而且界面
上完全看不出来——它只是"答得不太对"。
"""
from __future__ import annotations
import json
import pytest
from httpx import ASGITransport, AsyncClient
from app.api.conversations import HISTORY_TURNS, history_text, title_from
from app.db.base import SessionLocal
from app.db.models import Conversation, ConversationTurn
from app.main import app
@pytest.fixture
async def client():
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as c:
yield c
async def _conversation(client, **kw) -> dict:
r = await client.post("/api/conversations", json={"kind": "chat", **kw})
assert r.status_code == 201, r.text
return r.json()
async def _turn(client, conv_id: str, question: str, **patch) -> dict:
r = await client.post(f"/api/conversations/{conv_id}/turns", json={"question": question})
assert r.status_code == 201, r.text
turn = r.json()
if patch:
r = await client.patch(f"/api/conversations/{conv_id}/turns/{turn['id']}", json=patch)
assert r.status_code == 200, r.text
turn = r.json()
return turn
# --------------------------------------------------------------------------
# 会话本身
# --------------------------------------------------------------------------
async def test_a_conversation_survives_and_lists(client) -> None:
conv = await _conversation(client)
await _turn(client, conv["id"], "shop 里有几家店", answer="12 家", status="done")
# 列表里认得出是哪次聊天
rows = (await client.get("/api/conversations")).json()
mine = next(r for r in rows if r["id"] == conv["id"])
assert mine["turn_count"] == 1
assert mine["last_question"] == "shop 里有几家店"
# 完整取回:这是刷新页面后重建界面的唯一依据
detail = (await client.get(f"/api/conversations/{conv['id']}")).json()
assert [t["question"] for t in detail["turns"]] == ["shop 里有几家店"]
assert detail["turns"][0]["answer"] == "12 家"
async def test_title_comes_from_the_first_question(client) -> None:
"""一排"新对话"等于没有列表。"""
conv = await _conversation(client)
assert conv["title"] == "新对话"
await _turn(client, conv["id"], "shop 里有几家店")
assert (await client.get(f"/api/conversations/{conv['id']}")).json()["title"] == "shop 里有几家店"
# 第二轮不该再改标题
await _turn(client, conv["id"], "再按月份拆一下")
assert (await client.get(f"/api/conversations/{conv['id']}")).json()["title"] == "shop 里有几家店"
async def test_a_renamed_conversation_keeps_its_name(client) -> None:
conv = await _conversation(client)
await client.patch(f"/api/conversations/{conv['id']}", json={"title": "门店盘点"})
await _turn(client, conv["id"], "shop 里有几家店")
assert (await client.get(f"/api/conversations/{conv['id']}")).json()["title"] == "门店盘点"
def test_long_titles_are_cut_not_wrapped() -> None:
assert title_from("短问题") == "短问题"
assert title_from("") == "新对话"
long = title_from("请帮我统计一下过去十二个月每个门店每个月的订单总额并按同比排序")
assert len(long) <= 25 and long.endswith("…")
async def test_deleting_a_conversation_takes_its_turns(client) -> None:
conv = await _conversation(client)
turn = await _turn(client, conv["id"], "问点什么")
assert (await client.delete(f"/api/conversations/{conv['id']}")).status_code == 204
assert (await client.get(f"/api/conversations/{conv['id']}")).status_code == 404
async with SessionLocal() as session:
assert await session.get(ConversationTurn, turn["id"]) is None
async def test_archived_conversations_leave_the_list(client) -> None:
conv = await _conversation(client)
await client.patch(f"/api/conversations/{conv['id']}", json={"archived": True})
assert conv["id"] not in {r["id"] for r in (await client.get("/api/conversations")).json()}
rows = (await client.get("/api/conversations?include_archived=true")).json()
assert conv["id"] in {r["id"] for r in rows}
async def test_canvas_conversations_stay_out_of_the_chat_list(client) -> None:
"""画布的对话依附工作流,和问数据页不是一回事,混在一起反而难找。"""
canvas = await _conversation(client, kind="canvas")
assert canvas["id"] not in {r["id"] for r in (await client.get("/api/conversations")).json()}
rows = (await client.get("/api/conversations?kind=canvas")).json()
assert canvas["id"] in {r["id"] for r in rows}
async def test_a_failed_turn_is_still_recorded(client) -> None:
"""只记成功的轮次,历史就成了一份美化过的记录——而人回来最想搞清楚的
恰恰是上次卡在哪。"""
conv = await _conversation(client)
await _turn(client, conv["id"], "查点什么", status="error", error="模型不存在")
detail = (await client.get(f"/api/conversations/{conv['id']}")).json()
assert detail["turns"][0]["status"] == "error"
assert "模型不存在" in detail["turns"][0]["error"]
# --------------------------------------------------------------------------
# 给模型看的那一份
# --------------------------------------------------------------------------
GRAPH = {
"nodes": [
{"id": "ask", "type": "input", "position": {"x": 1, "y": 2}, "data": {"config": {}}},
{"id": "shop_agent", "type": "agent", "position": {"x": 9, "y": 9},
"data": {"label": "查门店", "config": {"tools": ["db_query__shop"]}}},
],
"edges": [{"source": "ask", "target": "shop_agent"}],
}
async def _seeded(client) -> str:
conv = await _conversation(client)
await _turn(client, conv["id"], "shop 里有几家店",
answer="一共 12 家", graph=GRAPH, status="done")
return conv["id"]
async def test_history_text_is_plain_qa(client) -> None:
conv_id = await _seeded(client)
async with SessionLocal() as session:
text = await history_text(session, conv_id)
assert "shop 里有几家店" in text and "一共 12 家" in text
async def test_history_carries_the_review_warning_forward(client) -> None:
"""被复核判过不可信的答案,下一轮必须知道。
只把正文喂过去的话,模型会拿一个已知有问题的结论当事实继续往下推
("那 12 家里有几家是新开的?"),错误就这么一轮轮传下去,而且越传
越像真的——后面几轮看起来都在正常作答。
"""
conv = await _conversation(client)
await _turn(
client, conv["id"], "shop 里有几家店",
answer="一共 12 家", status="done",
review={"verdict": "annotated", "severity": "broken", "note": "取数没跑通,这个数不可信。",
"answer": None, "retry": True, "signals": []},
)
async with SessionLocal() as session:
text = await history_text(session, conv["id"])
assert "一共 12 家" in text
assert "取数没跑通" in text
async def test_history_text_is_empty_without_a_conversation(client) -> None:
async with SessionLocal() as session:
assert await history_text(session, None) == ""
assert await history_text(session, (await _conversation(client))["id"]) == ""
async def test_the_turn_in_flight_is_not_its_own_history(client) -> None:
"""当前这一轮还没有答案,把它算进上下文只会得到一句自己的问题。"""
conv = await _conversation(client)
await _turn(client, conv["id"], "正在问的这句") # 保持 running
async with SessionLocal() as session:
assert await history_text(session, conv["id"]) == ""
async def test_history_keeps_only_the_recent_turns(client) -> None:
conv = await _conversation(client)
for i in range(HISTORY_TURNS + 3):
await _turn(client, conv["id"], f"第{i}问", answer=f"第{i}答", status="done")
async with SessionLocal() as session:
text = await history_text(session, conv["id"])
assert "第0问" not in text # 最早的挤出去了
assert f"第{HISTORY_TURNS + 2}问" in text
assert text.count("问:") == HISTORY_TURNS
async def test_a_huge_answer_does_not_crowd_out_the_other_turns(client) -> None:
conv = await _conversation(client)
await _turn(client, conv["id"], "给我全部数据", answer="x" * 50_000, status="done")
await _turn(client, conv["id"], "那总共多少", answer="12", status="done")
async with SessionLocal() as session:
text = await history_text(session, conv["id"])
assert len(text) < 2000
assert "那总共多少" in text # 后一轮没被前一轮挤掉
async def test_history_section_carries_the_previous_graph(client) -> None:
"""只给问答文字不够:追问会让模型重新设计一张图,可能接到别的表上。"""
from app.api.copilot import _history_section
conv_id = await _seeded(client)
async with SessionLocal() as session:
block = await _history_section(session, conv_id)
assert "shop 里有几家店" in block
assert "shop_agent" in block and "db_query__shop" in block
assert '"x"' not in block and '"position"' not in block # _slim 去掉了坐标噪音
# --------------------------------------------------------------------------
# 上下文真的到了模型手上
# --------------------------------------------------------------------------
async def test_the_previous_turn_reaches_the_prompt(client, monkeypatch) -> None:
"""整件事的关键一环。
拼字符串的函数测通过、却没接进 messages,是这类功能最典型的失败方式:
界面上完全看不出来,它只是"答得不太对"。所以这里拦在模型入口上,
看送出去的那条 human message 里到底有没有上一轮。
"""
import app.api.copilot as copilot
conv_id = await _seeded(client)
captured: list = []
class _Recorder:
def with_structured_output(self, *_a, **_k):
return self
async def ainvoke(self, messages):
captured.append(messages)
return {"nodes": [], "edges": []}
async def _model(*_a, **_k):
return _Recorder(), "mock-fast"
monkeypatch.setattr(copilot, "get_chat_model", _model)
r = await client.post("/api/copilot/generate", json={
"instruction": "再按月份拆一下", "intent": "answer", "conversation_id": conv_id,
})
assert r.status_code == 200, r.text
human = next(text for role, text in captured[0] if role == "human")
assert "shop 里有几家店" in human # 上一轮问的
assert "一共 12 家" in human # 上一轮答的
assert "shop_agent" in human # 上一轮的图
assert "再按月份拆一下" in human # 这一轮问的
async def test_no_conversation_means_no_history_header(client, monkeypatch) -> None:
"""不带 conversation_id 时不能凭空多出一个空的历史标题。"""
import app.api.copilot as copilot
captured: list = []
class _Recorder:
def with_structured_output(self, *_a, **_k):
return self
async def ainvoke(self, messages):
captured.append(messages)
return {"nodes": [], "edges": []}
async def _model(*_a, **_k):
return _Recorder(), "mock-fast"
monkeypatch.setattr(copilot, "get_chat_model", _model)
r = await client.post("/api/copilot/generate", json={
"instruction": "查一下门店", "intent": "answer",
})
assert r.status_code == 200, r.text
human = next(text for role, text in captured[0] if role == "human")
assert "之前聊过的" not in human
assert human.startswith("用户问了下面这个问题")
# --------------------------------------------------------------------------
# 追问不该把图搞丢
# --------------------------------------------------------------------------
class _Streamer:
"""按脚本吐操作行的假模型。"""
def __init__(self, lines: list[str]) -> None:
self._lines = lines
async def astream(self, _messages):
from langchain_core.messages import AIMessageChunk
for line in self._lines:
yield AIMessageChunk(content=line + "\n")
async def _stream_graph(client, monkeypatch, conv_id: str, lines: list[str]) -> dict:
"""跑一次 generate-stream,返回最后那条 final/error。"""
import app.api.copilot as copilot
async def _model(*_a, **_k):
return _Streamer(lines), "mock-fast"
monkeypatch.setattr(copilot, "get_chat_model", _model)
async with client.stream("POST", "/api/copilot/generate-stream", json={
"instruction": "哪六个?哪五个?", "intent": "answer", "conversation_id": conv_id,
}) as r:
last = {}
async for line in r.aiter_lines():
if line.startswith("data: "):
last = json.loads(line[6:])
return last
async def test_a_follow_up_that_only_sends_edits_still_produces_a_graph(
client, monkeypatch,
) -> None:
"""真踩过的那条:f5db9809 的第二轮。
上下文注入是对的——模型的说明写着"在上一轮已接好 Shop 数据源的 agent
节点上更新提示词",它完全看懂了。但它照着改图的协议只发了 update_node,
而这条路径的累加器从空图起步,于是那些操作全落空,用户收到的是
"图校验未通过:图是空的,先拖一个节点进来"——他只是问了句追问。
"""
conv_id = await _seeded(client)
final = await _stream_graph(client, monkeypatch, conv_id, [
'{"op":"plan","summary":"在上一轮的 agent 上改提示词"}',
'{"op":"update_node","id":"shop_agent","config":{"prompt":"列出具体名单"}}',
'{"op":"done","explanation":"沿用上一轮的图,只改了提示词"}',
])
assert final.get("op") == "final", final
ids = [n["id"] for n in final["graph"]["nodes"]]
assert ids, "重放到上一轮那张图上之后不该还是空的"
assert "shop_agent" in ids
# 改动要真的落上去,不是原样把旧图还回来
agent = next(n for n in final["graph"]["nodes"] if n["id"] == "shop_agent")
assert agent["data"]["config"]["prompt"] == "列出具体名单"
async def test_a_full_rebuild_does_not_drag_the_old_nodes_along(
client, monkeypatch,
) -> None:
"""模型照约定完整重发时,兜底不能画蛇添足。
否则上一轮的 shop_agent 会和新节点一起留在图里,变成一个没人连的孤儿。
"""
conv_id = await _seeded(client)
final = await _stream_graph(client, monkeypatch, conv_id, [
'{"op":"plan","summary":"重新搭"}',
'{"op":"add_node","node":{"id":"in","type":"input","label":"入口","config":{}}}',
'{"op":"add_node","node":{"id":"fresh_agent","type":"agent","label":"查","config":{}}}',
'{"op":"add_edge","edge":{"source":"in","target":"fresh_agent"}}',
'{"op":"add_node","node":{"id":"out","type":"output","label":"成果","config":{}}}',
'{"op":"add_edge","edge":{"source":"fresh_agent","target":"out"}}',
'{"op":"done","explanation":"新图"}',
])
assert final.get("op") == "final", final
ids = {n["id"] for n in final["graph"]["nodes"]}
assert ids == {"in", "fresh_agent", "out"}
assert "shop_agent" not in ids
async def test_the_first_turn_has_nothing_to_fall_back_on(client, monkeypatch) -> None:
"""没有上一轮时,空图就是空图——不能凭空变出一张来掩盖模型什么都没给。"""
conv = await _conversation(client)
final = await _stream_graph(client, monkeypatch, conv["id"], [
'{"op":"plan","summary":"想想"}',
'{"op":"done","explanation":"什么都没搭"}',
])
assert final["op"] == "final"
assert final["graph"]["nodes"] == []
assert any("空" in str(i.get("message", "")) for i in final["issues"]), final["issues"]
# --------------------------------------------------------------------------
# 不是每句话都值得跑一遍库
# --------------------------------------------------------------------------
#
# 以前必执行:final 一到就无条件 launch。于是「重试」「继续找找」这种对过程
# 说的话,也各自重建了一整张图、跑一遍 153 张表的库;而一句明确要「设计一个
# 工作流」的,建完也直接跑了——用户要的是那张图,不是这一次的结果。
async def test_a_question_about_earlier_turns_needs_no_workflow(client, monkeypatch) -> None:
"""reply 这条路径:不建图、不跑、也不该走到那套排版校验上。
走到了的话,空图会撞出"图校验未通过:图是空的,先拖一个节点进来"。
"""
conv_id = await _seeded(client)
final = await _stream_graph(client, monkeypatch, conv_id, [
'{"op":"reply","text":"第 1 轮走的是 role.level IN (platform, regional) 这个口径。"}',
])
assert final["op"] == "reply"
assert "口径" in final["text"]
assert "graph" not in final
async def test_asking_for_a_workflow_builds_it_but_does_not_run_it(client, monkeypatch) -> None:
"""用户要的是流程本身,跑一遍只是替他多花一次钱。"""
conv_id = await _seeded(client)
final = await _stream_graph(client, monkeypatch, conv_id, [
'{"op":"plan","summary":"搭一个每天能跑的"}',
'{"op":"add_node","node":{"id":"in","type":"input","label":"入口","config":{}}}',
'{"op":"add_node","node":{"id":"out","type":"output","label":"成果","config":{}}}',
'{"op":"add_edge","edge":{"source":"in","target":"out"}}',
'{"op":"done","explanation":"每天跑一次","run":false}',
])
assert final["op"] == "final"
assert final["autorun"] is False
assert [n["id"] for n in final["graph"]["nodes"]] == ["in", "out"]
async def test_a_normal_data_question_still_runs(client, monkeypatch) -> None:
"""默认不变:问数据就是建图 + 跑。没写 run 就当 true。"""
conv_id = await _seeded(client)
final = await _stream_graph(client, monkeypatch, conv_id, [
'{"op":"plan","summary":"查一下"}',
'{"op":"add_node","node":{"id":"in","type":"input","label":"入口","config":{}}}',
'{"op":"add_node","node":{"id":"out","type":"output","label":"成果","config":{}}}',
'{"op":"add_edge","edge":{"source":"in","target":"out"}}',
'{"op":"done","explanation":"跑完给结论"}',
])
assert final["op"] == "final"
assert final["autorun"] is True
def test_the_three_paths_are_spelled_out_for_the_model() -> None:
"""路径选择全靠这段约定,尤其是那条不许凭印象给数字的硬约束。"""
from app.api.copilot import GenerateIn, _STREAM_PROTOCOL, _user_message
text = _user_message(GenerateIn(instruction="随便问问", intent="answer"), patch=False)
assert "直接回答" in text and "建图并执行" in text and "建图但不执行" in text
assert "不得新断言任何数据事实" in text
# 协议里要给得出这两个出口,否则模型没法表达
assert '"op":"reply"' in _STREAM_PROTOCOL
assert '"run":true' in _STREAM_PROTOCOL
async def test_copilot_is_told_not_to_invent_a_memory_scope() -> None:
"""记忆域由平台给。图里自己起一个,写进去就再也读不回来。
真发生过(会话 cbe09f4b 的第 2~3 轮):一轮的 memory 节点写了
scope="user",下一轮没写 scope、落回 default,于是「我叫张三」存下来了,
问「我是谁」却答不上来——记忆在库里好好的,只是读错了地方。
"""
from app.api.copilot import NODE_REFERENCE
memory_line = next(
(l for l in NODE_REFERENCE.splitlines() if l.strip().startswith("- memory:")), "")
assert memory_line, "节点参考里找不到 memory 这一行"
assert "scope" not in memory_line, "scope 还列在可填字段里,模型就会去填它"
assert "不要写 scope" in NODE_REFERENCE
# --------------------------------------------------------------------------
# 主动记忆
# --------------------------------------------------------------------------
def test_copilot_is_told_when_to_record_a_memory() -> None:
"""以前只有用户明说「记住」才会写:17 轮里只有 4 轮带记忆节点,全在用户
主动谈记忆的那个会话里。于是他说过一次「我叫张三」,换个话题再问就
谁也不知道了。
但边界比指令本身更要紧——记错了会污染以后每一次对话。
"""
from app.api.copilot import GenerateIn, _user_message
text = _user_message(GenerateIn(instruction="随便问问", intent="answer"), patch=False)
assert "该记" in text and "不该记" in text
# 最容易被记进去的垃圾:这一轮的问题、查出来的数、一次性指令、以及推测
for junk in ("这一轮的问题本身", "查出来的数", "一次性的指令", "推测出来的东西"):
assert junk in text, f"没有拦住「{junk}」这类"
assert "拿不准就不记" in text
async def test_writing_a_memory_is_visible_in_the_trace(client, monkeypatch) -> None:
"""自动记忆的前提是看得见。
以前 memory 节点发的是 info 日志,而解码层丢弃所有 info——等于系统背着
用户往长期记忆里存东西。让 Copilot 主动记之后这条不可接受。
"""
from app.core.events import EventType
assert EventType.MEMORY_END == "memory.end"
# 节点确实发这条事件,而且带上了记了什么
import inspect
from app.engine.nodes import knowledge as node
src = inspect.getsource(node.run_memory)
assert "MEMORY_END" in src
assert "LOG, level=\"info\"" not in src, "又退回 info 日志了,界面上会看不见"
assert 'result.get("content"' in src, "没把记了什么带出去,只报条数等于没说"