mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 04:41:59 +00:00
cc608f56ec
「跑完了」和「答得对」之间一直没有任何一层。库里 264 条 succeeded 里, 14 条带着 warn、4 条 agent 步数用满,而交到用户手上的答案对此只字不提。 最糟的一次(run dd9927e6)交出去的干脆是「本轮只执行了第一个工具」—— 一句内部管道文案冒充结论。 出具体系(issuance.py)本来就是干这个的,但它挂在输出节点的可选契约上, 还要配口径卡提供受控指标集;问数据页生成的图基本都是 input → agent → output, 走不到那条路。所以补一层轻的。 **干净的运行一次模型都不调。** 先用规则扫一遍事件流(engine/review.py:scan), 没扫出东西就到此为止。实测 15ms 返回。这是整层设计的前提,测试里用 「monkeypatch 一个会抛异常的模型,断言它没被碰过」来守。 **分类挂在 code 上,不是中文消息串。** 12 处 warn 发射点都带上 code。 这样「流式失败,回退非流式」能归成 benign 直接跳过——它对答案内容零影响, 为它叫一次模型等于每次网络抖一下就多花一笔;而「步数用满」归 broken。 原来这两者在事件流里长得一模一样。工具报错之后同一个工具又跑成了, 算 degraded 不算 broken——否则参数自愈每修一次就报一次「答案不可信」。 **复核不许现编数字。** 它手上只有原答案和一份异常清单,没有任何原始数据, 写出来的新数字只能是凭空来的。直接复用 issuance.trace_numbers,把原答案的 数字当受控指标集,对不上就整条退回原答案、只留说明。硬拦截,不是提示。 **规则管重试资格,模型只有否决权。** 一开始写成两边取并,实测 loop_limit 时模型判 retry=true——而重跑一遍循环还是会撞上限,白烧一次建图加一次跑图。 反过来之后,「检索什么都没查到」这类也不会被模型强推重跑。 **重试带着失败的那张图,不是推倒重来。** 原来传 base_graph=null,模型只看到 「原问题 + 一句失败原因」,很可能原样再建出同一张坏图。而且它连自己上次建了 什么都看不到——会话历史按 status != running 过滤,正在重试的这一轮恰好被 排除在外。现在交回去走后端的改图分支,调大步数、改正参数就是改一个字段。 上限一次。 **答案等复核完成再显示。** 端到端抓到的:答案先摆出来,十秒后说明才追上去—— 这十秒里用户读的是一个没人核对过的结论,正是要修的那件事本身。改成落库立刻 (防关标签页丢结果)、显示等复核,并给复核加 25s 上限。说明排在成果上方, broken 用报警色、degraded 用普通边框——混成一个样子用户就学会了一概忽略。 复核结论跟着轮次落库,刷新后还在;也进下一轮的对话历史,否则模型会拿一个 已知不可信的结论当事实继续往下推。 验证:后端 242 passed(新增 test_review.py 25 条);tsc 干净; check-decode / check-stream / check-ui 全过,check-stream 新增 9 条渲染断言。 两条端到端都在真浏览器 + 真模型上跑通:检索降级那次说明正确排在答案上方; 重试那次建图请求 2 次、第二次 base_graph 带着失败的 3 个节点、第一版没漏出去。 复核结果不是事件流,decode.ts 翻译不到它,所以渲染测试放在 check-stream。
364 lines
14 KiB
Python
364 lines
14 KiB
Python
"""跑完之后的复核层。
|
|
|
|
核心不变量有两条,其余都是围着它们转的:
|
|
|
|
1. 干净的运行**一次模型都不调**。复核是补救手段,不是每次问数的固定开销。
|
|
2. 复核模型看不到原始数据,所以它写出来的任何新数字都是现编的,必须机械拦住。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
|
|
from app.core.events import EventType
|
|
from app.engine import review as rv
|
|
from app.main import app
|
|
|
|
|
|
@pytest.fixture
|
|
async def client():
|
|
transport = ASGITransport(app=app)
|
|
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
def ev(etype, **data):
|
|
return {"type": str(etype), "node_id": data.pop("node_id", ""), "data": data}
|
|
|
|
|
|
def warn(message, code=""):
|
|
return ev(EventType.LOG, level="warn", message=message, code=code)
|
|
|
|
|
|
OK_OUTPUT = {"结果": "管理员有 6 个。"}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# scan:规则层
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def test_clean_run_has_no_signals():
|
|
"""干净的运行扫不出东西——这是"零成本"的前提。"""
|
|
events = [
|
|
ev(EventType.RUN_STARTED),
|
|
ev(EventType.TOOL_START, tool="db_query__shop"),
|
|
ev(EventType.TOOL_END, tool="db_query__shop"),
|
|
ev(EventType.LOG, level="info", message="随便什么"),
|
|
ev(EventType.RETRIEVE_END, count=5, degraded=False, query="权限"),
|
|
ev(EventType.RUN_FINISHED),
|
|
]
|
|
assert rv.scan(events, OK_OUTPUT) == []
|
|
|
|
|
|
def test_tool_error_is_broken():
|
|
signals = rv.scan([ev(EventType.TOOL_ERROR, tool="db_query__x", error="no such table")],
|
|
OK_OUTPUT)
|
|
assert [s.kind for s in signals] == ["tool_error"]
|
|
assert signals[0].severity == rv.BROKEN
|
|
assert "no such table" in signals[0].detail
|
|
|
|
|
|
def test_tool_error_then_success_is_only_degraded():
|
|
"""自愈接住了就不该报"答案不可信"。
|
|
|
|
参数自动纠正那条路每修一次就发一条 tool.error,不区分的话每次自愈
|
|
都会触发一次复核,还告诉用户答案不可信——而它明明跑通了。
|
|
"""
|
|
signals = rv.scan([
|
|
ev(EventType.TOOL_ERROR, tool="db_query__x", error="unexpected keyword 'query'"),
|
|
ev(EventType.TOOL_END, tool="db_query__x"),
|
|
], OK_OUTPUT)
|
|
assert [s.kind for s in signals] == ["tool_args_fixed"]
|
|
assert signals[0].severity == rv.DEGRADED
|
|
|
|
|
|
def test_retrieve_degraded_and_empty():
|
|
degraded = rv.scan([ev(EventType.RETRIEVE_END, count=3, degraded=True)], OK_OUTPUT)
|
|
assert degraded[0].kind == "retrieve_degraded"
|
|
assert degraded[0].severity == rv.DEGRADED
|
|
|
|
empty = rv.scan([ev(EventType.RETRIEVE_END, count=0, query="商家账户")], OK_OUTPUT)
|
|
assert empty[0].kind == "retrieve_empty"
|
|
assert empty[0].severity == rv.BROKEN
|
|
assert "商家账户" in empty[0].detail
|
|
|
|
|
|
def test_specific_retrieve_reason_wins_over_the_generic_one():
|
|
"""检索降级会发两条:knowledge.py 先发具体原因的 warn,再发 retrieve.end。
|
|
|
|
两边都收就是同一件事报两遍,用户读到一条含糊的加一条具体的,还得自己
|
|
判断是不是两个问题。
|
|
"""
|
|
signals = rv.scan([
|
|
warn("default 里有 32 条向量由「local-hashing」建立,和当前的对不上", "retrieve_degraded"),
|
|
ev(EventType.RETRIEVE_END, count=5, degraded=True),
|
|
], OK_OUTPUT)
|
|
assert len(signals) == 1
|
|
assert "32 条向量" in signals[0].detail
|
|
|
|
|
|
def test_node_and_run_failures_are_broken():
|
|
signals = rv.scan([
|
|
ev(EventType.NODE_FAILED, node_id="agent1", error="boom"),
|
|
ev(EventType.RUN_FAILED, error="超时"),
|
|
], OK_OUTPUT)
|
|
assert {s.kind for s in signals} == {"node_failed", "run_failed"}
|
|
assert all(s.severity == rv.BROKEN for s in signals)
|
|
|
|
|
|
def test_warn_classified_by_code_not_by_message():
|
|
"""分类挂在 code 上。消息是给人读的,随时会改。"""
|
|
assert rv.scan([warn("agent 用满了 8 步还没给出结论。", "step_limit")], OK_OUTPUT)[0] \
|
|
.severity == rv.BROKEN
|
|
assert rv.scan([warn("第 1 次校验失败:不是合法 JSON", "validate_retry")], OK_OUTPUT)[0] \
|
|
.severity == rv.DEGRADED
|
|
|
|
|
|
def test_benign_warns_do_not_trigger_review():
|
|
"""流式回退、隔离降级对答案内容零影响。
|
|
|
|
为它们叫一次模型,等于每次网络抖一下就多花一次调用,还要用户读一句
|
|
与结论无关的告警。轨迹里有记录就够了。
|
|
"""
|
|
events = [
|
|
warn("流式失败,回退非流式:connection reset", "stream_fallback"),
|
|
warn("节点要求 strict 隔离,但 microVM 未就绪,实际用了 seatbelt", "isolation_fallback"),
|
|
]
|
|
assert rv.scan(events, OK_OUTPUT) == []
|
|
|
|
|
|
def test_warn_without_code_defaults_to_degraded():
|
|
"""库里的老事件没有 code。宁可多说一句,也好过把真问题当噪音吞掉。"""
|
|
signals = rv.scan([warn("某个没带 code 的告警")], OK_OUTPUT)
|
|
assert [s.kind for s in signals] == ["warn"]
|
|
assert signals[0].severity == rv.DEGRADED
|
|
|
|
|
|
def test_empty_and_placeholder_output_are_broken():
|
|
assert rv.scan([], None)[0].kind == "empty_output"
|
|
assert rv.scan([], {"结果": ""})[0].kind == "empty_output"
|
|
|
|
# run dd9927e6 真的把这句内部管道文案当答案交出去过
|
|
leaked = {"结果": "本轮只执行了第一个工具。看到它的结果之后,再决定这一个还要不要调。"}
|
|
assert rv.scan([], leaked)[0].kind == "placeholder_output"
|
|
|
|
|
|
def test_duplicate_signals_collapse():
|
|
"""一次运行里同一条告警发五遍,用户只需要看到一次。"""
|
|
events = [warn("第 1 次校验失败:不是合法 JSON", "validate_retry")] * 5
|
|
assert len(rv.scan(events, OK_OUTPUT)) == 1
|
|
|
|
|
|
def test_scan_accepts_orm_like_rows():
|
|
"""事件既可能是 dict 也可能是 ORM 行,两种都要吃得下。"""
|
|
|
|
class Row:
|
|
type = str(EventType.TOOL_ERROR)
|
|
node_id = "n1"
|
|
data = {"tool": "t", "error": "e"}
|
|
|
|
assert rv.scan([Row()], OK_OUTPUT)[0].kind == "tool_error"
|
|
|
|
|
|
def test_worst_and_should_retry():
|
|
broken = rv.Signal("step_limit", "步数用满", rv.BROKEN)
|
|
degraded = rv.Signal("retrieve_degraded", "检索降级", rv.DEGRADED)
|
|
assert rv.worst([degraded, broken]) == rv.BROKEN
|
|
assert rv.worst([degraded]) == rv.DEGRADED
|
|
assert rv.worst([]) == ""
|
|
|
|
assert rv.should_retry([broken]) is True
|
|
# 检索什么都没查到,重跑一遍还是查不到——那不是重跑能解决的
|
|
assert rv.should_retry([rv.Signal("retrieve_empty", "没命中", rv.BROKEN)]) is False
|
|
assert rv.should_retry([degraded]) is False
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# check_numbers:不得新增数字
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def test_check_numbers_allows_reordering():
|
|
original = "管理员有 6 个,商家有 5 个,占比 45.5%。"
|
|
revised = "占比 45.5%。其中商家 5 个、管理员 6 个。"
|
|
assert rv.check_numbers(original, revised) == []
|
|
|
|
|
|
def test_check_numbers_catches_invented():
|
|
invented = rv.check_numbers("管理员有 6 个。", "管理员有 6 个,商家有 5 个。")
|
|
assert invented == ["5"]
|
|
|
|
|
|
def test_check_numbers_ignores_prose_without_numbers():
|
|
assert rv.check_numbers("管理员有 6 个。", "这次没能查到有效数据,建议重试。") == []
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# review:调模型那一段
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
class FakeModel:
|
|
"""最小的 with_structured_output 替身。calls 记录被调了几次。"""
|
|
|
|
def __init__(self, reply):
|
|
self.reply = reply
|
|
self.calls = 0
|
|
|
|
def with_structured_output(self, schema):
|
|
return self
|
|
|
|
async def ainvoke(self, messages):
|
|
self.calls += 1
|
|
if isinstance(self.reply, Exception):
|
|
raise self.reply
|
|
return self.reply
|
|
|
|
|
|
SIGNALS = [rv.Signal("step_limit", "agent 用满了 8 步还没给出结论", rv.BROKEN)]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_review_annotates_without_rewriting():
|
|
model = FakeModel({"note": "这次没跑完就收尾了。", "answer": "", "retry": True})
|
|
result = await rv.review(model, question="有几个管理员", answer="管理员有 6 个。",
|
|
signals=SIGNALS)
|
|
assert result.verdict == "annotated"
|
|
assert result.answer is None # 沿用原答案
|
|
assert result.note == "这次没跑完就收尾了。"
|
|
assert result.retry is True
|
|
assert result.severity == rv.BROKEN
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_review_adopts_clean_rewrite():
|
|
model = FakeModel({
|
|
"note": "步数用满了。", "answer": "管理员有 6 个(这个数可能不完整)。", "retry": False,
|
|
})
|
|
result = await rv.review(model, question="q", answer="管理员有 6 个。", signals=SIGNALS)
|
|
assert result.verdict == "rewritten"
|
|
assert result.answer == "管理员有 6 个(这个数可能不完整)。"
|
|
# 规则层管资格、模型有否决权:step_limit 本来够格重跑,模型说不必就不跑
|
|
assert result.retry is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_model_cannot_force_an_unhelpful_retry():
|
|
"""检索什么都没查到,重跑一遍还是查不到——模型说要重跑也不能听。
|
|
|
|
否则就是白烧一次建图加一次跑图,而用户等的时间翻倍。
|
|
"""
|
|
model = FakeModel({"note": "知识库里没有这个。", "answer": "", "retry": True})
|
|
result = await rv.review(
|
|
model, question="q", answer="没查到。",
|
|
signals=[rv.Signal("retrieve_empty", "知识库里没有检索到相关内容", rv.BROKEN)],
|
|
)
|
|
assert result.severity == rv.BROKEN
|
|
assert result.retry is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_review_reverts_rewrite_with_invented_numbers():
|
|
"""复核手上只有原答案,它写出来的新数字只能是凭空来的。"""
|
|
model = FakeModel({
|
|
"note": "步数用满了。", "answer": "管理员有 6 个,商家有 5 个。", "retry": False,
|
|
})
|
|
result = await rv.review(model, question="q", answer="管理员有 6 个。", signals=SIGNALS)
|
|
assert result.verdict == "reverted"
|
|
assert result.answer is None # 那份正文一个字都不能用
|
|
assert "5" in result.note # 但要说清是为什么退的
|
|
assert "步数用满了。" in result.note # 说明本身还是要给
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_review_survives_model_failure():
|
|
"""复核只是加一层说明,它自己挂了不能把已经到手的答案也带走。"""
|
|
model = FakeModel(RuntimeError("502"))
|
|
result = await rv.review(model, question="q", answer="管理员有 6 个。", signals=SIGNALS)
|
|
assert result.verdict == "annotated"
|
|
assert result.answer is None
|
|
assert result.note == SIGNALS[0].detail # 退回规则层自己的措辞
|
|
assert result.retry is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_review_survives_garbage_reply():
|
|
model = FakeModel("这不是 JSON,也不是对象")
|
|
result = await rv.review(model, question="q", answer="管理员有 6 个。", signals=SIGNALS)
|
|
assert result.verdict == "annotated"
|
|
assert result.note
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 接口
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def _make_run(client, output, events):
|
|
"""直接落一条运行和它的事件,省掉真跑一遍图。"""
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import Run, RunEvent
|
|
|
|
async with SessionLocal() as session:
|
|
run = Run(workflow_id=None, status="succeeded", output=output, graph={})
|
|
session.add(run)
|
|
await session.flush()
|
|
for i, e in enumerate(events):
|
|
session.add(RunEvent(
|
|
run_id=run.id, seq=i + 1, type=e["type"],
|
|
node_id=e["node_id"] or None, ts=0.0, data=e["data"],
|
|
))
|
|
await session.commit()
|
|
return run.id
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_clean_run_never_touches_the_model(client, monkeypatch):
|
|
"""干净运行零成本——这条是整层设计的前提,要用「模型根本没被碰过」来断言。"""
|
|
async def explode(*a, **k):
|
|
raise AssertionError("干净的运行不该调模型")
|
|
|
|
monkeypatch.setattr("app.providers.factory.get_chat_model", explode)
|
|
monkeypatch.setattr("app.api.copilot.get_chat_model", explode)
|
|
|
|
run_id = await _make_run(client, OK_OUTPUT, [
|
|
ev(EventType.TOOL_START, tool="t"), ev(EventType.TOOL_END, tool="t"),
|
|
])
|
|
resp = await client.post("/api/copilot/review", json={"run_id": run_id, "question": "q"})
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["verdict"] == "ok"
|
|
assert body["signals"] == []
|
|
assert body["answer"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_review_endpoint_reports_signals(client, monkeypatch):
|
|
"""没配模型时也得把规则层扫到的东西说出来,而不是静默放行。"""
|
|
from app.providers.factory import ProviderNotConfigured
|
|
|
|
async def unconfigured(*a, **k):
|
|
raise ProviderNotConfigured("没有可用的 provider")
|
|
|
|
monkeypatch.setattr("app.api.copilot.get_chat_model", unconfigured)
|
|
|
|
run_id = await _make_run(client, OK_OUTPUT, [
|
|
ev(EventType.TOOL_ERROR, tool="db_query__x", error="no such table: users"),
|
|
])
|
|
body = (await client.post(
|
|
"/api/copilot/review", json={"run_id": run_id, "question": "q"}
|
|
)).json()
|
|
assert body["verdict"] == "annotated"
|
|
assert body["severity"] == rv.BROKEN
|
|
assert [s["kind"] for s in body["signals"]] == ["tool_error"]
|
|
assert "no such table" in body["note"]
|
|
assert body["retry"] is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_review_endpoint_404(client):
|
|
resp = await client.post("/api/copilot/review", json={"run_id": "没这个", "question": "q"})
|
|
assert resp.status_code == 404
|