Files
AgentLab/backend/tests/test_notify_events.py
T
ThearchyHelios 2371e75f17 评测执行的主干:后台评测、无头作答、评分、统计、对比与 A/B、清理(下一轮第 3 波 W3-c 第一段)
为什么:第 2 波只有题集和期望答案的来源,评测本身跑不起来。这一段把细案 01 做法 9–12、16、17 的后端接上,
量的是用户在问数据页拿到的同一轮,而不是另抄一份编排。

- eval/answer.py:无头作答,串起 answer_ops → chat/turn 的决定 → launch_run(origin=eval,带目录视图、只读、遮罩)
  → 等终态(停在审批的取消、记「评测不代批」;驳回记路径「被驳回」)→ review_of_run → 按同一规则重来一次;
  工作流题按钉住版本的图跑探索运行(不发正式运行,免得被 pick_base 和应用视图取到)。用量按角色记
- eval/runner.py:用 core/job_runtime,一次评测一个任务,题 × 重复 × 臂一项;A/B 几臂共用期望、同一题同时开跑;
  开评拍目录快照、记上传表格版本和表结构哈希的前后;并发不超过 max_parallel 和全局并发的四分之一;每题时限;
  评测单独一本账(开跑前估算、每题预留后找补、触顶不再开新题);止损;能停、只重试没做完的、重启标中断不续跑
- eval/score.py:六种期望、五种结论(量纲不同单列)、维度配对与弱匹配、路径不对直接不通过、口径 dual 两版都报;
  逐题 E1 用口径账本的 compare_queries(总方案 X2,不另写 signals);SCORER_VERSION 可重评(eval/rescore.py,只读封存)
- eval/judging.py:结论裁判走按需裁判同一条路(evidence._judge_now),文字要点由裁判判、单列不计准确率;花费经
  judge.charged_to 记在评测那本账上(engine/judge.py 只加了一个上下文变量和取账本的一处)
- eval/stats.py、eval/compare.py:准确率与可追溯准确率 + Wilson 区间、各项分布、P50/P90、按角色成本、引用率、逐题 E1;
  选基准、逐题转移与归因、指纹差异、A/B 配对表与精确 McNemar;「目录贡献」预设对照(当前 vs 关,可加只用已核实)
- eval/cleanup.py + core/artifact_store.forget:保留最近 N 次 + 钉住的基线 + A/B 另几臂,删结果、明细工件、底下的运行和
  引用行,工件文件一条引用都不剩才删;总表永久保留
- engine/tolerance.py:双盲复算的 _agree / _delta_pct 原样搬来共用(dual_check 改 import,行为不变)
- 设置加 eval 一组(builder 读、admin 写)和 GET /api/settings/eval/spend;题集预算在全站缺省上覆盖,每日上限只能往下收
- api/evals.py:发起(单臂 / A/B / 目录贡献预设)、估算、进度、逐题、停止、重试、重评、钉基线、和上一次比、配对表、清理
- main.py 的 lifespan:影子起草和评测的启动整理、关停(评测先于运行停,停时取消自己发起的运行)
- 测试:test_eval_e2e(dual_check_shop + 剧本模型:按支付日 330 通过且 SQL 等价,按下单日 310 不通过、差在 time;
  目录对照 1:0;和上一次比找到退步、归因做法变);通知事件、定时接口形状的老测试按新事件和新字段更新
2026-10-16 16:38:09 +08:00

171 lines
8.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""通知事件的登记表(app/notify/service.EVENTS;总方案 X6)。
本轮新增的六个事件(数据源变化、口径召回、评测退步 / 失败、待办汇总、密钥到期)在第 1 波一次登记好:名字、界面上的叫法、
推给哪几条路都在 EVENTS 一处,前端的叫法在 lib/terms.ts 的 NOTIFY_TEXT 一处。第 2、3 波实现推送的那一块只改自己那一行
的范围,不再去改同一个元组、同一张叫法表(几块并行改同一处必然冲突)。这里守:
- 登记表:六个事件都在,叫法不空;推给哪几条路是空的就是「登记了、还没有推送」——接口不收这种订阅、GET /api/notify/events
照实给空的 scopes,设置页和应用的通知里都不列(勾了收不到就是「看着生效、其实没生效」,个人渠道「实例完成」的老问题);
- 前后端对照:terms.ts 的 eventLabel 和 EVENTS 一一对应、逐字一致;投递记录才有的事件(改派、升级、催办、测试消息)另有一张
叫法表,和服务端 _AS 对上(写法同 test_chat_turn_policy.py,两边一改就对不上);
- 推给哪几条路是真的:_channels 只走事件登记了的路;没登记的事件推不出去,直接报错;
- 订阅的范围不对(全局渠道勾只推个人的事件)拒收,但只核对这次新勾的:以前存下的照旧,开关、改名不会因此 422。
渠道名、地址都是虚构的。
"""
from __future__ import annotations
import re
from pathlib import Path
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import delete
from app.core.crypto import encrypt
from app.db.base import SessionLocal
from app.db.models import NotifyChannel
from app.notify import service
TERMS_TS = Path(__file__).resolve().parents[2] / "frontend" / "src" / "lib" / "terms.ts"
URL = "https://hooks.example.com/agentlab/notify-events-5d1c"
#: 本轮登记的六个(总方案 X6)
THIS_ROUND = ("source.changed", "ledger.recall", "eval.regressed", "eval.failed", "task.digest", "key.expiring")
#: 其中已经推送的(第 3 波:定时评测的通知只推全局渠道,notify_eval),其余还没有
PUSHED_THIS_ROUND = {"eval.regressed": {"global"}, "eval.failed": {"global"}}
@pytest.fixture
async def client():
from app.main import app
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
async with SessionLocal() as session:
await session.execute(delete(NotifyChannel).where(NotifyChannel.name.like("事件登记-%")))
await session.commit()
# --------------------------------------------------------------------------
# 登记表
# --------------------------------------------------------------------------
def test_this_rounds_events_are_registered_once_and_not_pushed_yet():
for name in THIS_ROUND:
assert name in service.EVENTS, f"{name} 没有登记"
assert service.EVENTS[name].label, name
if name in PUSHED_THIS_ROUND:
continue
assert service.EVENTS[name].scopes == (), f"{name} 还没有推送:不该有人订阅得到"
assert len({e.label for e in service.EVENTS.values()}) == len(service.EVENTS), "两个事件叫成了同一句"
assert service.EVENT_LABEL == {k: v.label for k, v in service.EVENTS.items()}
def test_events_that_are_pushed_today_keep_their_audience():
"""现有五个事件推给哪几条路,照 _channels 和各个 notify_* 现在的做法登记(行为不变)。"""
every = {"global", "user", "app"}
assert {k: set(v.scopes) for k, v in service.EVENTS.items() if v.scopes} == {
"task.opened": every, "run.finished": every, "run.failed": every, "action.unknown": every,
# 发给提交人自己(和订阅了它的全局渠道),不归哪个应用
"review.done": {"global", "user"},
**PUSHED_THIS_ROUND,
}
for event in service.EVENTS.values():
assert set(event.scopes) <= every
def _object_block(source: str, name: str) -> str:
"""terms.ts 里 NOTIFY_TEXT 下面名叫 name 的那个对象字面量(到配对的右花括号为止)。"""
head = source.index("export const NOTIFY_TEXT = {")
start = source.index(f"{name}: {{", head) + len(name) + 2
depth, i = 1, start + 1
while depth:
depth += {"{": 1, "}": -1}.get(source[i], 0)
i += 1
return source[start:i]
def _entries(block: str) -> dict[str, str]:
return dict(re.findall(r"'?([a-z]+(?:\.[a-z_]+)?)'?:\s*'([^']*)'", block))
def test_frontend_labels_match_the_registry_word_for_word():
source = TERMS_TS.read_text(encoding="utf-8")
labels = _entries(_object_block(source, "eventLabel"))
assert labels == service.EVENT_LABEL, "设置页的叫法和服务端登记的对不上(一种说法)"
delivery = _entries(_object_block(source, "deliveryEventLabel"))
assert set(delivery) == set(service._AS) | {"test"}, "投递记录里会出现的事件都要有叫法"
assert not set(delivery) & set(labels)
# --------------------------------------------------------------------------
# 接口
# --------------------------------------------------------------------------
async def test_events_endpoint_says_who_receives_each_event(client):
rows = (await client.get("/api/notify/events")).json()
assert [r["event"] for r in rows] == list(service.EVENTS)
by = {r["event"]: r for r in rows}
assert by["eval.regressed"] == {"event": "eval.regressed", "label": "评测有退步", "scopes": ["global"]}
assert by["ledger.recall"]["scopes"] == []
assert by["review.done"]["scopes"] == ["global", "user"]
async def test_events_not_pushed_yet_cannot_be_subscribed(client):
resp = await client.post("/api/notify/channels", json={
"name": "事件登记-评测", "url": URL, "events": ["task.opened", "ledger.recall"]})
assert resp.status_code == 422, resp.text
detail = resp.json()["detail"]
assert "口径升版,答案的数变了" in detail and "还没有推送" in detail
unknown = await client.post("/api/notify/channels", json={"name": "事件登记-未知", "url": URL, "events": ["nope"]})
assert unknown.status_code == 422
assert "口径升版,答案的数变了" not in unknown.json()["detail"], "可选的里面不列还没有推送的"
assert "待办打开" in unknown.json()["detail"]
async def test_scope_mismatch_is_refused_only_for_newly_ticked_events(client):
refused = await client.post("/api/notify/channels", json={
"name": "事件登记-应用", "scope": "app", "url": URL, "events": ["review.done"]})
assert refused.status_code == 422, refused.text
assert "我提交审核的应用有结果" in refused.json()["detail"] and "应用" in refused.json()["detail"]
# 以前存下的(当时界面对哪种渠道都列这一项):开关、改名照旧,不因此 422
async with SessionLocal() as session:
row = NotifyChannel(name="事件登记-旧", scope="app", url=encrypt(URL), events=["task.opened", "review.done"])
session.add(row)
await session.commit()
channel_id = row.id
toggled = await client.patch(f"/api/notify/channels/{channel_id}", json={"enabled": False})
assert toggled.status_code == 200, toggled.text
assert toggled.json()["events"] == ["task.opened", "review.done"]
again = await client.patch(f"/api/notify/channels/{channel_id}", json={"events": ["review.done", "run.failed"]})
assert again.status_code == 200, again.text
added = await client.patch(f"/api/notify/channels/{channel_id}", json={"events": ["review.done", "eval.failed"]})
assert added.status_code == 422 and "评测失败" in added.json()["detail"]
# --------------------------------------------------------------------------
# 推送只走登记了的路
# --------------------------------------------------------------------------
async def test_channels_only_walk_the_paths_the_event_is_registered_for(monkeypatch):
monkeypatch.setitem(service.EVENTS, "demo.personal", service.Event("只推个人的演示事件", ("user",)))
async with SessionLocal() as session:
row = NotifyChannel(name="事件登记-全局", scope="global", url=encrypt(URL),
events=["demo.personal", "task.opened", "ledger.recall"])
session.add(row)
await session.commit()
try:
ids = lambda found: [c.id for c in found] # noqa: E731
assert row.id not in ids(await service._channels(session, "demo.personal", None, [])), "全局渠道这条路没登记"
assert row.id in ids(await service._channels(session, "task.opened", None, []))
assert await service._channels(session, "ledger.recall", None, []) == [], "还没有推送的:谁都不推"
with pytest.raises(ValueError, match="没有登记"):
await service._channels(session, "demo.unregistered", None, [])
finally:
await session.delete(row)
await session.commit()