Files
AgentLab/backend/tests/test_eval_schedule.py
T
ThearchyHelios e29adf80d0 评测执行的测试:执行、评分、统计与对比、清理、定时与通知
为什么:照细案 01 的测试清单把这一波的行为钉住,顺带修了跑测试时撞出来的两处。

- test_eval_runner:后台排队、并发不超过 max_parallel 也不超过全局并发的四分之一、同一题集同时只能一次、停止后只重做
  没做完的(评完的不动)、重启标中断不续跑、单次金额上限触顶不再开新题、评测每日账单独记(不占裁判、核查的额度)、
  连续 3 题助手调用失败止损、每题超时取消正在跑的运行、停在审批上的作答取消掉(评测不代批)、接口的估算与进度、设置里
  eval 一组和 /settings/eval/spend
- test_eval_score:五种结论、写出精度和百分数、维度配对与弱匹配、多行 / 名单 / 没有数据 / 反问、路径和期望先于数字、
  dual 两版、允许的相对误差和双盲复算是同一个函数;读封存的答案、逐题 E1 用账本的比对函数;重评只读封存、不调模型
- test_eval_stats:Wilson、精确 McNemar、P50/P90 和手算对照;总表的各项;转移与四种归因;指纹差异;配对表;选基准
- test_eval_cleanup:保留规则;forget 只删一条引用都不剩的文件;清理后总表还在、结果和底下的运行删掉、共用的工件留着
- test_eval_schedule:定时接口收题集(拒工作流、输入模板、Webhook)、到点建一次评测并写 eval_run_id、重叠跳过、错过的按
  宽限补;有退步推「评测有退步」、没评完推「评测失败」,消息里没有期望值和答案里的数
- test_eval_e2e 补:开评拍目录快照、记表结构前后;打开裁判时结论裁判走按需裁判那条路、花费记在评测账上,文字要点单列
- 修:期望反问的题在建图、运行失败(含超时)时把原因写全;改非工作流目标的定时任务时提交后不再读过期对象
2026-10-16 16:49:47 +08:00

228 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""评测题集的定时与通知(app/eval/scheduled.py、notify/service.notify_eval;细案 01 做法 15,总方案 X4、X6)。
- 定时接口按 target_kind 收评测题集:核对目标交给 app/eval/scheduled.check_target(题集在、没归档、有启用的题);非工作流
目标不收工作流、输入模板、Webhook(Webhook 的事件说的是那一次运行,非工作流目标收不到,照实拒);接口返回目标的种类、id、
名字;工作流的定时行为一字不变(test_schedules_api.py);
- 到点建一次评测:定时跑 1 次,截至时间是计划触发时间,ScheduleFire.eval_run_id 指向它;上一次还没结束就 skipped_overlap;
错过的按宽限补(同一套调度循环);
- 有一道题从通过变成不通过推「评测有退步」,没评完推「评测失败」:只推订阅了的全局渠道,消息里只有题集名、通过数、退步题数、
链接,不放期望值和答案里的数。题集名、渠道都是虚构的。
"""
from __future__ import annotations
import asyncio
import json
import uuid
from datetime import datetime, timedelta, timezone
import httpx
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import delete, select
from app.core.crypto import encrypt
from app.db.base import SessionLocal, utcnow
from app.db.models import NotifyChannel, Schedule, ScheduleFire
from app.db.models_eval import EvalResult, EvalRun
from app.engine.runner import run_manager
from app.eval import answer as answer_mod
from app.eval import runner
from app.main import app
from app.notify import core
from app.notify import service as notify
from app.scheduler import loop, service
from tests import eval_samples as es
UTC = timezone.utc
URL = "https://hooks.example.com/agentlab/eval-7c2a"
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
async with SessionLocal() as session:
await session.execute(delete(Schedule))
await session.commit()
yield
await runner.manager.shutdown()
await notify.drain()
async with SessionLocal() as session:
await session.execute(delete(Schedule))
await session.execute(delete(NotifyChannel).where(NotifyChannel.name.like("评测通知-%")))
await session.commit()
await es.drop_suites()
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
@pytest.fixture
def received(monkeypatch):
got: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
got.append(request)
return httpx.Response(200, json={"ok": True})
monkeypatch.setattr(core, "_transport", httpx.MockTransport(handler))
monkeypatch.setattr(core, "RETRY_DELAYS", (0, 0, 0))
return got
def stub(monkeypatch, *, delay: float = 0.0, path: str = "asked"):
async def drive(self):
await asyncio.sleep(delay)
self.turn.path = path
return self.turn
monkeypatch.setattr(answer_mod.Driver, "drive", drive)
ASK = [{"question": "含糊的问题", "expectations": [{"kind": "ask"}]}]
async def _schedule(client, suite_id: str, **extra) -> dict:
made = await client.post("/api/schedules", json={"name": "每周回归", "target_kind": "eval_suite",
"target_id": suite_id, "cron": "0 9 * * 1",
"timezone": "Asia/Shanghai", **extra})
assert made.status_code == 201, made.text
return made.json()
async def test_the_schedule_api_takes_an_eval_suite(client):
suite_id = await es.make_suite("none", name="虚构回归题集", cases=ASK)
body = await _schedule(client, suite_id)
assert (body["target_kind"], body["target_id"], body["target_name"]) == ("eval_suite", suite_id, "虚构回归题集")
assert body["target_label"] == "评测题集" and body["workflow_id"] == "" and body["webhook"] is None
assert body["input_template"] == {} and body["target_exists"] is True
listed = (await client.get("/api/schedules")).json()["items"]
assert [s["target_name"] for s in listed] == ["虚构回归题集"]
for bad, field in (({"target_id": "nope"}, "target_id"), ({"target_id": None}, "target_id"),
({"webhook": {"url": URL}}, "webhook"), ({"input_template": {"a": "b"}}, "input_template"),
({"workflow_id": "wf"}, "workflow_id"), ({"target_kind": "nope"}, "target_kind")):
refused = await client.post("/api/schedules", json={"name": "x", "target_kind": "eval_suite",
"target_id": suite_id, "cron": "0 9 * * 1",
"timezone": "Asia/Shanghai", **bad})
assert refused.status_code == 422 and refused.json()["field"] == field, (bad, refused.text)
# 改名、改时间、停用都照常;目标种类建好不能改
patched = await client.patch(f"/api/schedules/{body['id']}", json={"name": "每天回归", "cron": "0 9 * * *",
"enabled": False, "webhook": None,
"version": body["version"]})
assert patched.status_code == 200, patched.text
assert patched.json()["name"] == "每天回归" and patched.json()["next_run_at"] is None
moved = await client.patch(f"/api/schedules/{body['id']}", json={"target_kind": "workflow",
"version": patched.json()["version"]})
assert moved.status_code == 422 and moved.json()["field"] == "target_kind"
# 题集归档了:照实拒
archived = await es.make_suite("none", name="归档的题集", cases=ASK)
async with SessionLocal() as session:
from app.db.models_eval import EvalSuite
(await session.get(EvalSuite, archived)).archived = True
await session.commit()
refused = await client.post("/api/schedules", json={"name": "x", "target_kind": "eval_suite", "target_id": archived,
"cron": "0 9 * * 1", "timezone": "Asia/Shanghai"})
assert refused.status_code == 422 and "已归档" in refused.json()["detail"]
async def test_firing_starts_one_evaluation_and_skips_on_overlap(client, monkeypatch):
stub(monkeypatch, delay=0.5)
suite_id = await es.make_suite("none", cases=ASK, default_variant={"repeats": 3})
body = await _schedule(client, suite_id)
now = utcnow().replace(microsecond=0)
fire = await service.fire_once(body["id"], now, "manual", actor="王芳")
assert fire.status == "started" and fire.eval_run_id and fire.run_id is None
async with SessionLocal() as session:
run = await session.get(EvalRun, fire.eval_run_id)
assert run.trigger == "schedule" and run.schedule_id == body["id"] and run.repeats == 1, "定时只跑 1 次"
assert run.as_of.replace(tzinfo=UTC) == now and run.created_by == "王芳"
again = await service.fire_once(body["id"], now + timedelta(seconds=1), "manual")
assert again.status == "skipped_overlap" and f"#{run.id[:6]}" in again.reason
same = await service.fire_once(body["id"], now, "manual")
assert same.id == fire.id, "同一次触发只发起一次"
await es.wait_eval(run.id)
history = (await client.get(f"/api/schedules/{body['id']}/fires")).json()["items"]
assert {f["eval_run_id"] for f in history if f["status"] == "started"} == {run.id}
async def test_a_missed_fire_within_grace_is_made_up(client, monkeypatch):
stub(monkeypatch)
suite_id = await es.make_suite("none", cases=ASK)
body = await _schedule(client, suite_id)
due = utcnow().replace(microsecond=0) - timedelta(minutes=20)
async with SessionLocal() as session:
await session.execute(Schedule.__table__.update().where(Schedule.id == body["id"]).values(next_run_at=due))
await session.commit()
fires = await loop.fire_due(due + timedelta(minutes=20))
[made_up] = [f for f in fires if f.status == "started"]
assert made_up.kind == "misfire" and made_up.eval_run_id
await es.wait_eval(made_up.eval_run_id)
async def _channel(events: list[str]) -> str:
async with SessionLocal() as session:
row = NotifyChannel(name=f"评测通知-{uuid.uuid4().hex[:6]}", scope="global", format="generic", url=encrypt(URL),
events=events)
session.add(row)
await session.commit()
return row.id
async def test_a_regression_is_pushed_without_numbers(client, monkeypatch, received, tmp_path):
name, sid = await es.make_source(tmp_path)
es.use_copilot(monkeypatch, es.Copilot(name))
es.use_report(monkeypatch)
suite_id = await es.make_suite(sid, name="虚构周回归")
body = await _schedule(client, suite_id)
await _channel(["eval.regressed", "eval.failed"])
first = await service.fire_once(body["id"], utcnow().replace(microsecond=0) - timedelta(minutes=5), "manual")
await es.wait_eval(first.eval_run_id)
assert received == [], "第一次没有可比的、也评完了:不推"
await es.set_notes(sid, {"label": es.NOTES["label"]})
second = await service.fire_once(body["id"], utcnow().replace(microsecond=0), "manual")
await es.wait_eval(second.eval_run_id)
[request] = received
message = json.loads(request.content)
assert message["event"] == "eval.regressed" and message["title"] == "评测「虚构周回归」有退步"
text = message["text"]
assert "通过:0 / 1 题" in text and "退步:1 题" in text
for secret in ("330", "310", "gmv", "orders", "paid_date"):
assert secret not in json.dumps(message, ensure_ascii=False), f"通知里不该有 {secret}"
from app.db.models import NotifyDelivery
async with SessionLocal() as session:
[delivery] = (await session.execute(select(NotifyDelivery).where(
NotifyDelivery.subject_id == second.eval_run_id))).scalars().all()
assert delivery.subject_kind == "eval_run" and delivery.status == "ok"
from tests.fixtures.sources import drop_source
await drop_source(sid)
async def test_an_unfinished_scheduled_evaluation_pushes_failed(client, monkeypatch, received):
async def drive(self):
self.turn.path, self.turn.copilot_failed, self.turn.reason = "build_failed", True, "助手本轮未完成"
return self.turn
monkeypatch.setattr(answer_mod.Driver, "drive", drive)
suite_id = await es.make_suite("none", cases=ASK * 4, name="虚构失败题集", budget={"max_parallel": 1})
body = await _schedule(client, suite_id)
await _channel(["eval.failed"])
fire = await service.fire_once(body["id"], utcnow().replace(microsecond=0), "manual")
await es.wait_eval(fire.eval_run_id)
async with SessionLocal() as session:
run = await session.get(EvalRun, fire.eval_run_id)
skipped = (await session.execute(select(EvalResult).where(
EvalResult.eval_run_id == run.id, EvalResult.status == "skipped"))).scalars().all()
assert run.status == "partial" and run.stop_reason.startswith("连续 3 题") and len(skipped) == 1
[request] = received
message = json.loads(request.content)
assert message["event"] == "eval.failed" and message["title"] == "评测「虚构失败题集」没有评完"
assert "通过:0 / 3 题" in message["text"]
# 链接去那条定时任务的触发历史(设了对外访问地址才有链接)
assert all("/schedules/" in link["url"] for link in message["links"])